You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@apex.apache.org by th...@apache.org on 2017/03/07 06:58:06 UTC
[01/30] apex-malhar git commit: Renamed demos to examples. Packages
and artifactid names are changed as suggested.
Repository: apex-malhar
Updated Branches:
refs/heads/master 4b36bf3e5 -> d5bf96cac
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/wordcount/src/main/java/org/apache/apex/examples/wordcount/LineReader.java
----------------------------------------------------------------------
diff --git a/examples/wordcount/src/main/java/org/apache/apex/examples/wordcount/LineReader.java b/examples/wordcount/src/main/java/org/apache/apex/examples/wordcount/LineReader.java
new file mode 100644
index 0000000..25336ae
--- /dev/null
+++ b/examples/wordcount/src/main/java/org/apache/apex/examples/wordcount/LineReader.java
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.wordcount;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.Path;
+
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.annotation.OutputPortFieldAnnotation;
+import com.datatorrent.lib.io.fs.AbstractFileInputOperator;
+
+/**
+ * Reads lines from input file and returns them. If EOF is reached, a control tuple
+ * is emitted on the control port
+ *
+ * @since 3.2.0
+ */
+public class LineReader extends AbstractFileInputOperator<String>
+{
+ private static final Logger LOG = LoggerFactory.getLogger(LineReader.class);
+
+ /**
+ * Output port on which lines from current file name are emitted
+ */
+ public final transient DefaultOutputPort<String> output = new DefaultOutputPort<>();
+
+ /**
+ * Control port on which the current file name is emitted to indicate EOF
+ */
+ @OutputPortFieldAnnotation(optional = true)
+ public final transient DefaultOutputPort<String> control = new DefaultOutputPort<>();
+
+ private transient BufferedReader br = null;
+
+ private Path path;
+
+ /**
+ * File open callback; wrap the file input stream in a buffered reader for reading lines
+ * @param curPath The path to the file just opened
+ */
+ @Override
+ protected InputStream openFile(Path curPath) throws IOException
+ {
+ LOG.info("openFile: curPath = {}", curPath);
+ path = curPath;
+ InputStream is = super.openFile(path);
+ br = new BufferedReader(new InputStreamReader(is));
+ return is;
+ }
+
+ /**
+ * File close callback; close buffered reader
+ * @param is File input stream that will imminently be closed
+ */
+ @Override
+ protected void closeFile(InputStream is) throws IOException
+ {
+ super.closeFile(is);
+ br.close();
+ br = null;
+ path = null;
+ }
+
+ /**
+ * {@inheritDoc}
+ * If we hit EOF, emit file name on control port
+ */
+ @Override
+ protected String readEntity() throws IOException
+ {
+ // try to read a line
+ final String line = br.readLine();
+ if (null != line) { // common case
+ LOG.debug("readEntity: line = {}", line);
+ return line;
+ }
+
+ // end-of-file; send control tuple, containing only the last component of the path
+ // (only file name) on control port
+ //
+ if (control.isConnected()) {
+ LOG.info("readEntity: EOF for {}", path);
+ final String name = path.getName(); // final component of path
+ control.emit(name);
+ }
+
+ return null;
+ }
+
+ @Override
+ protected void emit(String tuple)
+ {
+ output.emit(tuple);
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/wordcount/src/main/java/org/apache/apex/examples/wordcount/WCPair.java
----------------------------------------------------------------------
diff --git a/examples/wordcount/src/main/java/org/apache/apex/examples/wordcount/WCPair.java b/examples/wordcount/src/main/java/org/apache/apex/examples/wordcount/WCPair.java
new file mode 100644
index 0000000..a960695
--- /dev/null
+++ b/examples/wordcount/src/main/java/org/apache/apex/examples/wordcount/WCPair.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.wordcount;
+
+/**
+ * A single (word, frequency) pair
+ *
+ * @since 3.2.0
+ */
+public class WCPair
+{
+ /**
+ * The word
+ */
+ public String word;
+
+ /**
+ * The frequency
+ */
+ public int freq;
+
+ /**
+ * Default constructor
+ */
+ public WCPair()
+ {
+
+ }
+
+ /**
+ * Create new object with given values
+ * @param w The word
+ * @param f The frequency
+ */
+ public WCPair(String w, int f)
+ {
+ word = w;
+ freq = f;
+ }
+
+ @Override
+ public String toString()
+ {
+ return String.format("(%s, %d)", word, freq);
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/wordcount/src/main/java/org/apache/apex/examples/wordcount/WindowWordCount.java
----------------------------------------------------------------------
diff --git a/examples/wordcount/src/main/java/org/apache/apex/examples/wordcount/WindowWordCount.java b/examples/wordcount/src/main/java/org/apache/apex/examples/wordcount/WindowWordCount.java
new file mode 100644
index 0000000..09c3921
--- /dev/null
+++ b/examples/wordcount/src/main/java/org/apache/apex/examples/wordcount/WindowWordCount.java
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.wordcount;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.datatorrent.api.DefaultInputPort;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.common.util.BaseOperator;
+
+/**
+ * Computes word frequencies per window and emits them at each {@code endWindow()}. The output is a
+ * list of (word, frequency) pairs
+ *
+ * @since 3.2.0
+ */
+public class WindowWordCount extends BaseOperator
+{
+ private static final Logger LOG = LoggerFactory.getLogger(WindowWordCount.class);
+
+ /** {@literal (word => frequency)} map for current window */
+ protected Map<String, WCPair> wordMap = new HashMap<>();
+
+ /**
+ * Input port on which words are received
+ */
+ public final transient DefaultInputPort<String> input = new DefaultInputPort<String>()
+ {
+ @Override
+ public void process(String word)
+ {
+ WCPair pair = wordMap.get(word);
+ if (null != pair) { // word seen previously
+ pair.freq += 1;
+ return;
+ }
+
+ // new word
+ pair = new WCPair();
+ pair.word = word;
+ pair.freq = 1;
+ wordMap.put(word, pair);
+ }
+ };
+
+ /**
+ * Output port which emits the list of word frequencies for current window
+ */
+ public final transient DefaultOutputPort<List<WCPair>> output = new DefaultOutputPort<>();
+
+ /**
+ * {@inheritDoc}
+ * If we've seen some words in this window, emit the map and clear it for next window
+ */
+ @Override
+ public void endWindow()
+ {
+ LOG.info("WindowWordCount: endWindow");
+
+ // got EOF; if no words found, do nothing
+ if (wordMap.isEmpty()) {
+ return;
+ }
+
+ // have some words; emit single map and reset for next file
+ final ArrayList<WCPair> list = new ArrayList<>(wordMap.values());
+ output.emit(list);
+ list.clear();
+ wordMap.clear();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/wordcount/src/main/java/org/apache/apex/examples/wordcount/WordCountInputOperator.java
----------------------------------------------------------------------
diff --git a/examples/wordcount/src/main/java/org/apache/apex/examples/wordcount/WordCountInputOperator.java b/examples/wordcount/src/main/java/org/apache/apex/examples/wordcount/WordCountInputOperator.java
new file mode 100644
index 0000000..d5fc66a
--- /dev/null
+++ b/examples/wordcount/src/main/java/org/apache/apex/examples/wordcount/WordCountInputOperator.java
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.wordcount;
+
+import java.io.BufferedReader;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.datatorrent.lib.io.SimpleSinglePortInputOperator;
+
+/**
+ * <p>WordCountInputOperator class.</p>
+ *
+ * @since 0.3.2
+ */
+public class WordCountInputOperator extends SimpleSinglePortInputOperator<String> implements Runnable
+{
+
+ private static final Logger logger = LoggerFactory.getLogger(WordCountInputOperator.class);
+ protected long averageSleep = 300;
+ protected long sleepPlusMinus = 100;
+ protected String fileName = "com/datatorrent/examples/wordcount/samplefile.txt";
+
+ public void setAverageSleep(long as)
+ {
+ averageSleep = as;
+ }
+
+ public void setSleepPlusMinus(long spm)
+ {
+ sleepPlusMinus = spm;
+ }
+
+ public void setFileName(String fn)
+ {
+ fileName = fn;
+ }
+
+ @Override
+ public void run()
+ {
+ BufferedReader br = null;
+ DataInputStream in = null;
+ InputStream fstream = null;
+
+ while (true) {
+ try {
+ String line;
+ fstream = this.getClass().getClassLoader().getResourceAsStream(fileName);
+
+ in = new DataInputStream(fstream);
+ br = new BufferedReader(new InputStreamReader(in));
+
+ while ((line = br.readLine()) != null) {
+ String[] words = line.trim().split("[\\p{Punct}\\s\\\"\\'\u201c\u201d]+");
+ for (String word : words) {
+ word = word.trim().toLowerCase();
+ if (!word.isEmpty()) {
+ outputPort.emit(word);
+ }
+ }
+ try {
+ Thread.sleep(averageSleep + (new Double(sleepPlusMinus * (Math.random() * 2 - 1))).longValue());
+ } catch (InterruptedException ex) {
+ // nothing
+ }
+ }
+
+ } catch (IOException ex) {
+ logger.debug(ex.toString());
+ } finally {
+ try {
+ if (br != null) {
+ br.close();
+ }
+ if (in != null) {
+ in.close();
+ }
+ if (fstream != null) {
+ fstream.close();
+ }
+ } catch (IOException exc) {
+ // nothing
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/wordcount/src/main/java/org/apache/apex/examples/wordcount/WordCountWriter.java
----------------------------------------------------------------------
diff --git a/examples/wordcount/src/main/java/org/apache/apex/examples/wordcount/WordCountWriter.java b/examples/wordcount/src/main/java/org/apache/apex/examples/wordcount/WordCountWriter.java
new file mode 100644
index 0000000..e4ee35e
--- /dev/null
+++ b/examples/wordcount/src/main/java/org/apache/apex/examples/wordcount/WordCountWriter.java
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.wordcount;
+
+import java.io.UnsupportedEncodingException;
+
+import java.util.List;
+import java.util.Map;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.datatorrent.lib.io.fs.AbstractFileOutputOperator;
+
+/**
+ * Write top N words and their frequencies to a file
+ *
+ * @since 3.2.0
+ */
+public class WordCountWriter extends AbstractFileOutputOperator<Map<String, Object>>
+{
+ private static final Logger LOG = LoggerFactory.getLogger(WordCountWriter.class);
+ private static final String charsetName = "UTF-8";
+ private static final String nl = System.lineSeparator();
+
+ private String fileName; // current file name
+ private final transient StringBuilder sb = new StringBuilder();
+
+ /**
+ * {@inheritDoc}
+ * Invoke requestFinalize() to create the output file with the desired name without decorations.
+ */
+ @Override
+ public void endWindow()
+ {
+ if (null != fileName) {
+ requestFinalize(fileName);
+ }
+ super.endWindow();
+ }
+
+ /**
+ * Extracts file name from argument
+ * @param tuple Singleton map {@literal (fileName => L) where L is a list of (word, frequency) pairs}
+ * @return the file name to write the tuple to
+ */
+ @Override
+ protected String getFileName(Map<String, Object> tuple)
+ {
+ LOG.info("getFileName: tuple.size = {}", tuple.size());
+
+ final Map.Entry<String, Object> entry = tuple.entrySet().iterator().next();
+ fileName = entry.getKey();
+ LOG.info("getFileName: fileName = {}", fileName);
+ return fileName;
+ }
+
+ /**
+ * Extracts output file content from argument
+ * @param tuple Singleton map {@literal (fileName => L) where L is a list of (word, frequency) pairs}
+ * @return input tuple converted to an array of bytes
+ */
+ @Override
+ protected byte[] getBytesForTuple(Map<String, Object> tuple)
+ {
+ LOG.info("getBytesForTuple: tuple.size = {}", tuple.size());
+
+ // get first and only pair; key is the fileName and is ignored here
+ final Map.Entry<String, Object> entry = tuple.entrySet().iterator().next();
+ final List<WCPair> list = (List<WCPair>)entry.getValue();
+
+ if (sb.length() > 0) { // clear buffer
+ sb.delete(0, sb.length());
+ }
+
+ for ( WCPair pair : list ) {
+ sb.append(pair.word);
+ sb.append(" : ");
+ sb.append(pair.freq);
+ sb.append(nl);
+ }
+
+ final String data = sb.toString();
+ LOG.info("getBytesForTuple: data = {}", data);
+ try {
+ final byte[] result = data.getBytes(charsetName);
+ return result;
+ } catch (UnsupportedEncodingException ex) {
+ throw new RuntimeException("Should never get here", ex);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/wordcount/src/main/java/org/apache/apex/examples/wordcount/WordReader.java
----------------------------------------------------------------------
diff --git a/examples/wordcount/src/main/java/org/apache/apex/examples/wordcount/WordReader.java b/examples/wordcount/src/main/java/org/apache/apex/examples/wordcount/WordReader.java
new file mode 100644
index 0000000..663d399
--- /dev/null
+++ b/examples/wordcount/src/main/java/org/apache/apex/examples/wordcount/WordReader.java
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.wordcount;
+
+import java.util.regex.Pattern;
+
+import com.datatorrent.api.Context.OperatorContext;
+import com.datatorrent.api.DefaultInputPort;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.common.util.BaseOperator;
+
+/**
+ * Extracts words from input line
+ *
+ * @since 3.3.0
+ */
+public class WordReader extends BaseOperator
+{
+ // default pattern for word-separators
+ private static final Pattern nonWordDefault = Pattern.compile("[\\p{Punct}\\s]+");
+
+ private String nonWordStr; // configurable regex
+ private transient Pattern nonWord; // compiled regex
+
+ /**
+ * Output port on which words from the current file are emitted
+ */
+ public final transient DefaultOutputPort<String> output = new DefaultOutputPort<>();
+
+ /**
+ * Input port on which lines from the current file are received
+ */
+ public final transient DefaultInputPort<String> input = new DefaultInputPort<String>()
+ {
+
+ @Override
+ public void process(String line)
+ {
+ // line; split it into words and emit them
+ final String[] words = nonWord.split(line);
+ for (String word : words) {
+ if (word.isEmpty()) {
+ continue;
+ }
+ output.emit(word);
+ }
+ }
+ };
+
+ /**
+ * Returns the regular expression that matches strings between words
+ * @return Regular expression for strings that separate words
+ */
+ public String getNonWordStr()
+ {
+ return nonWordStr;
+ }
+
+ /**
+ * Sets the regular expression that matches strings between words
+ * @param regex New regular expression for strings that separate words
+ */
+ public void setNonWordStr(String regex)
+ {
+ nonWordStr = regex;
+ }
+
+ /**
+ * {@inheritDoc}
+ * Set nonWord to the default pattern if necessary
+ */
+ @Override
+ public void setup(OperatorContext context)
+ {
+ if (null == nonWordStr) {
+ nonWord = nonWordDefault;
+ } else {
+ nonWord = Pattern.compile(nonWordStr);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/wordcount/src/main/java/org/apache/apex/examples/wordcount/doc-files/UniqueWordCounter.jpg
----------------------------------------------------------------------
diff --git a/examples/wordcount/src/main/java/org/apache/apex/examples/wordcount/doc-files/UniqueWordCounter.jpg b/examples/wordcount/src/main/java/org/apache/apex/examples/wordcount/doc-files/UniqueWordCounter.jpg
new file mode 100644
index 0000000..054baed
Binary files /dev/null and b/examples/wordcount/src/main/java/org/apache/apex/examples/wordcount/doc-files/UniqueWordCounter.jpg differ
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/wordcount/src/main/java/org/apache/apex/examples/wordcount/package-info.java
----------------------------------------------------------------------
diff --git a/examples/wordcount/src/main/java/org/apache/apex/examples/wordcount/package-info.java b/examples/wordcount/src/main/java/org/apache/apex/examples/wordcount/package-info.java
new file mode 100644
index 0000000..c394d0e
--- /dev/null
+++ b/examples/wordcount/src/main/java/org/apache/apex/examples/wordcount/package-info.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/**
+ * Streaming word count demonstration application.
+ */
+package org.apache.apex.examples.wordcount;
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/wordcount/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/examples/wordcount/src/main/resources/META-INF/properties.xml b/examples/wordcount/src/main/resources/META-INF/properties.xml
new file mode 100644
index 0000000..3d9e356
--- /dev/null
+++ b/examples/wordcount/src/main/resources/META-INF/properties.xml
@@ -0,0 +1,98 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<configuration>
+ <!-- TopNWordsWithQueries -->
+
+ <!-- for debugging -->
+ <!--
+ <property>
+ <name>dt.attr.CONTAINER_JVM_OPTIONS</name>
+ <value>-Dlog4j.configuration=my_log4j.properties</value>
+ </property>
+ -->
+
+ <!-- monitored input directory -->
+ <property>
+ <name>dt.application.TopNWordsWithQueries.operator.lineReader.directory</name>
+ <value>/tmp/test/input-dir</value>
+ </property>
+
+ <!-- regular expression for word separator -->
+ <property>
+ <name>dt.application.TopNWordsWithQueries.operator.wordReader.nonWordStr</name>
+ <value>[\p{Punct}\s]+</value>
+ </property>
+
+ <!-- output directory for word counts -->
+ <property>
+ <name>dt.application.TopNWordsWithQueries.operator.wcWriter.filePath</name>
+ <value>/tmp/test/output-dir</value>
+ </property>
+
+ <!-- Top N value -->
+ <property>
+ <name>dt.application.TopNWordsWithQueries.operator.fileWordCount.topN</name>
+ <value>10</value>
+ </property>
+
+ <!-- topic for queries (current file) -->
+ <property>
+ <name>dt.application.TopNWordsWithQueries.operator.snapshotServerFile.embeddableQueryInfoProvider.topic</name>
+ <value>TopNWordsQueryFile</value>
+ </property>
+
+ <!-- topic for query results (current file) -->
+ <property>
+ <name>dt.application.TopNWordsWithQueries.operator.wsResultFile.topic</name>
+ <value>TopNWordsQueryFileResult</value>
+ </property>
+
+ <!-- topic for queries (global) -->
+ <property>
+ <name>dt.application.TopNWordsWithQueries.operator.snapshotServerGlobal.embeddableQueryInfoProvider.topic</name>
+ <value>TopNWordsQueryGlobal</value>
+ </property>
+
+ <!-- topic for query results (global) -->
+ <property>
+ <name>dt.application.TopNWordsWithQueries.operator.wsResultGlobal.topic</name>
+ <value>TopNWordsQueryGlobalResult</value>
+ </property>
+
+ <!-- retry count -->
+ <property>
+ <name>dt.application.TwitterExample.operator.wsResult.numRetries</name>
+ <value>2147483647</value>
+ </property>
+
+
+ <!-- WordCountExample -->
+ <property>
+ <name>dt.application.WordCountExample.operator.wordinput.fileName</name>
+ <value>samplefile.txt</value>
+ </property>
+ <property>
+ <name>dt.application.WordCountExample.stream.wordinput.count.locality</name>
+ <value>CONTAINER_LOCAL</value>
+ <description>Specify container locality for the viewtuplecount stream
+ </description>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/wordcount/src/main/resources/WordDataSchema.json
----------------------------------------------------------------------
diff --git a/examples/wordcount/src/main/resources/WordDataSchema.json b/examples/wordcount/src/main/resources/WordDataSchema.json
new file mode 100644
index 0000000..5e8e7c0
--- /dev/null
+++ b/examples/wordcount/src/main/resources/WordDataSchema.json
@@ -0,0 +1,4 @@
+{
+ "values": [{"name": "word", "type": "string"},
+ {"name": "count", "type": "integer"}]
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/wordcount/src/main/resources/org/apache/apex/examples/wordcount/samplefile.txt
----------------------------------------------------------------------
diff --git a/examples/wordcount/src/main/resources/org/apache/apex/examples/wordcount/samplefile.txt b/examples/wordcount/src/main/resources/org/apache/apex/examples/wordcount/samplefile.txt
new file mode 100644
index 0000000..83eaaed
--- /dev/null
+++ b/examples/wordcount/src/main/resources/org/apache/apex/examples/wordcount/samplefile.txt
@@ -0,0 +1 @@
+CONTENT DELETED
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/wordcount/src/main/resources/samplefile.txt
----------------------------------------------------------------------
diff --git a/examples/wordcount/src/main/resources/samplefile.txt b/examples/wordcount/src/main/resources/samplefile.txt
new file mode 100644
index 0000000..02a5e70
--- /dev/null
+++ b/examples/wordcount/src/main/resources/samplefile.txt
@@ -0,0 +1,2 @@
+CONTENT DELETED
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/wordcount/src/test/java/org/apache/apex/examples/wordcount/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/wordcount/src/test/java/org/apache/apex/examples/wordcount/ApplicationTest.java b/examples/wordcount/src/test/java/org/apache/apex/examples/wordcount/ApplicationTest.java
new file mode 100644
index 0000000..f245fb8
--- /dev/null
+++ b/examples/wordcount/src/test/java/org/apache/apex/examples/wordcount/ApplicationTest.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.wordcount;
+
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.hadoop.conf.Configuration;
+import com.datatorrent.api.LocalMode;
+
+/**
+ *
+ */
+public class ApplicationTest
+{
+ private final transient Logger LOG = LoggerFactory.getLogger(ApplicationTest.class);
+ public ApplicationTest()
+ {
+ }
+
+ @Test
+ public void testSomeMethod() throws Exception
+ {
+ LocalMode lma = LocalMode.newInstance();
+ Configuration conf = new Configuration(false);
+ conf.addResource("dt-site-wordcount.xml");
+ lma.prepareDAG(new Application(), conf);
+ LocalMode.Controller lc = lma.getController();
+ long start = System.currentTimeMillis();
+ lc.run(300000);
+ long end = System.currentTimeMillis();
+ long time = end - start;
+ LOG.debug("Test used " + time + " ms");
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/wordcount/src/test/resources/dt-site-wordcount.xml
----------------------------------------------------------------------
diff --git a/examples/wordcount/src/test/resources/dt-site-wordcount.xml b/examples/wordcount/src/test/resources/dt-site-wordcount.xml
new file mode 100644
index 0000000..b7373e8
--- /dev/null
+++ b/examples/wordcount/src/test/resources/dt-site-wordcount.xml
@@ -0,0 +1,37 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<configuration>
+ <property>
+ <name>dt.application.WordCountExample.class</name>
+ <value>org.apache.apex.examples.wordcount.Application</value>
+ <description>An alias for the application</description>
+ </property>
+ <property>
+ <name>dt.application.WordCountExample.operator.wordinput.fileName</name>
+ <value>samplefile.txt</value>
+ </property>
+ <property>
+ <name>dt.application.WordCountExample.stream.wordinput.count.locality</name>
+ <value>CONTAINER_LOCAL</value>
+ <description>Specify container locality for the viewtuplecount stream
+ </description>
+ </property>
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/wordcount/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/examples/wordcount/src/test/resources/log4j.properties b/examples/wordcount/src/test/resources/log4j.properties
new file mode 100644
index 0000000..cf0d19e
--- /dev/null
+++ b/examples/wordcount/src/test/resources/log4j.properties
@@ -0,0 +1,43 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+log4j.rootLogger=DEBUG,CONSOLE
+
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+log4j.appender.CONSOLE.threshold=${test.log.console.threshold}
+test.log.console.threshold=DEBUG
+
+log4j.appender.RFA=org.apache.log4j.RollingFileAppender
+log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
+log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+log4j.appender.RFA.File=/tmp/app.log
+
+# to enable, add SYSLOG to rootLogger
+log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender
+log4j.appender.SYSLOG.syslogHost=127.0.0.1
+log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout
+log4j.appender.SYSLOG.layout.conversionPattern=${dt.cid} %-5p [%t] %c{2} %x - %m%n
+log4j.appender.SYSLOG.Facility=LOCAL1
+
+log4j.logger.org=info
+#log4j.logger.org.apache.commons.beanutils=warn
+log4j.logger.com.datatorrent=debug
+log4j.logger.org.apache.apex=debug
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/yahoofinance/pom.xml
----------------------------------------------------------------------
diff --git a/examples/yahoofinance/pom.xml b/examples/yahoofinance/pom.xml
new file mode 100644
index 0000000..c607198
--- /dev/null
+++ b/examples/yahoofinance/pom.xml
@@ -0,0 +1,65 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>malhar-examples-yahoo-finance</artifactId>
+ <packaging>jar</packaging>
+
+ <name>Apache Apex Malhar Yahoo! Finance Example</name>
+ <description>Apex example applications that get Yahoo finance feed and calculate minute price range, minute volume and simple moving average.</description>
+
+ <parent>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-examples</artifactId>
+ <version>3.7.0-SNAPSHOT</version>
+ </parent>
+
+ <properties>
+ <skipTests>true</skipTests>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>net.sf.opencsv</groupId>
+ <artifactId>opencsv</artifactId>
+ <version>2.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.derby</groupId>
+ <artifactId>derby</artifactId>
+ <version>10.9.1.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-contrib</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/yahoofinance/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/examples/yahoofinance/src/assemble/appPackage.xml b/examples/yahoofinance/src/assemble/appPackage.xml
new file mode 100644
index 0000000..4138cf2
--- /dev/null
+++ b/examples/yahoofinance/src/assemble/appPackage.xml
@@ -0,0 +1,59 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+ <id>appPackage</id>
+ <formats>
+ <format>jar</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>${basedir}/target/</directory>
+ <outputDirectory>/app</outputDirectory>
+ <includes>
+ <include>${project.artifactId}-${project.version}.jar</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/target/deps</directory>
+ <outputDirectory>/lib</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/site/conf</directory>
+ <outputDirectory>/conf</outputDirectory>
+ <includes>
+ <include>*.xml</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/META-INF</directory>
+ <outputDirectory>/META-INF</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/app</directory>
+ <outputDirectory>/app</outputDirectory>
+ </fileSet>
+ </fileSets>
+
+</assembly>
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/yahoofinance/src/main/java/org/apache/apex/examples/yahoofinance/ApplicationWithDerbySQL.java
----------------------------------------------------------------------
diff --git a/examples/yahoofinance/src/main/java/org/apache/apex/examples/yahoofinance/ApplicationWithDerbySQL.java b/examples/yahoofinance/src/main/java/org/apache/apex/examples/yahoofinance/ApplicationWithDerbySQL.java
new file mode 100644
index 0000000..517ef68
--- /dev/null
+++ b/examples/yahoofinance/src/main/java/org/apache/apex/examples/yahoofinance/ApplicationWithDerbySQL.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.yahoofinance;
+
+import org.apache.apex.malhar.contrib.misc.streamquery.AbstractSqlStreamOperator;
+import org.apache.apex.malhar.contrib.misc.streamquery.DerbySqlStreamOperator;
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.lib.io.ConsoleOutputOperator;
+
+/**
+ * This example will output the stock market data from yahoo finance
+ *
+ * @since 0.3.2
+ */
+@ApplicationAnnotation(name = "YahooFinanceWithDerbySQLExample")
+public class ApplicationWithDerbySQL implements StreamingApplication
+{
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ String symbolStr = conf.get(ApplicationWithDerbySQL.class.getName() + ".tickerSymbols", "YHOO,GOOG,AAPL,FB,AMZN,NFLX,IBM");
+
+ String[] symbols = symbolStr.split(",");
+
+ YahooFinanceCSVInputOperator input1 = dag.addOperator("input1", new YahooFinanceCSVInputOperator());
+ YahooFinanceCSVInputOperator input2 = dag.addOperator("input2", new YahooFinanceCSVInputOperator());
+ DerbySqlStreamOperator sqlOper = dag.addOperator("sqlOper", new DerbySqlStreamOperator());
+ ConsoleOutputOperator consoleOperator = dag.addOperator("console", new ConsoleOutputOperator());
+
+ for (String symbol : symbols) {
+ input1.addSymbol(symbol);
+ input2.addSymbol(symbol);
+ }
+ input1.addFormat("s0");
+ input1.addFormat("l1");
+ input2.addFormat("s0");
+ input2.addFormat("e0");
+ input2.addFormat("b4");
+
+ AbstractSqlStreamOperator.InputSchema inputSchema1 = new AbstractSqlStreamOperator.InputSchema("t1");
+ AbstractSqlStreamOperator.InputSchema inputSchema2 = new AbstractSqlStreamOperator.InputSchema("t2");
+ inputSchema1.setColumnInfo("s0", "varchar(100)", true); // symbol
+ inputSchema1.setColumnInfo("l1", "float", false); // last trade
+ inputSchema2.setColumnInfo("s0", "varchar(100)", true); // symbol
+ inputSchema2.setColumnInfo("e0", "float", false); // EPS
+ inputSchema2.setColumnInfo("b4", "float", false); // Book value
+
+ sqlOper.setInputSchema(0, inputSchema1);
+ sqlOper.setInputSchema(1, inputSchema2);
+
+ // Calculate PE Ratio and PB Ratio using SQL
+ sqlOper.addExecStatementString("SELECT SESSION.t1.s0 AS symbol, SESSION.t1.l1 / SESSION.t2.e0 AS pe_ratio, SESSION.t1.l1 / SESSION.t2.b4 AS pb_ratio FROM SESSION.t1,SESSION.t2 WHERE SESSION.t1.s0 = SESSION.t2.s0");
+
+ dag.addStream("input1_sql", input1.outputPort, sqlOper.in1);
+ dag.addStream("input2_sql", input2.outputPort, sqlOper.in2);
+
+ dag.addStream("result_console", sqlOper.result, consoleOperator.input);
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/yahoofinance/src/main/java/org/apache/apex/examples/yahoofinance/StockTickInput.java
----------------------------------------------------------------------
diff --git a/examples/yahoofinance/src/main/java/org/apache/apex/examples/yahoofinance/StockTickInput.java b/examples/yahoofinance/src/main/java/org/apache/apex/examples/yahoofinance/StockTickInput.java
new file mode 100644
index 0000000..cd31e03
--- /dev/null
+++ b/examples/yahoofinance/src/main/java/org/apache/apex/examples/yahoofinance/StockTickInput.java
@@ -0,0 +1,192 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.yahoofinance;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+
+import javax.validation.constraints.NotNull;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.httpclient.HttpClient;
+import org.apache.commons.httpclient.HttpStatus;
+import org.apache.commons.httpclient.cookie.CookiePolicy;
+import org.apache.commons.httpclient.methods.GetMethod;
+import org.apache.commons.httpclient.params.DefaultHttpParams;
+import org.apache.hadoop.util.StringUtils;
+
+import com.datatorrent.api.Context.OperatorContext;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.InputOperator;
+import com.datatorrent.api.annotation.OutputPortFieldAnnotation;
+import com.datatorrent.lib.util.KeyValPair;
+
+import au.com.bytecode.opencsv.CSVReader;
+
+/**
+ * This operator sends price, volume and time into separate ports and calculates incremental volume.
+ *
+ * @since 0.3.2
+ */
+public class StockTickInput implements InputOperator
+{
+ private static final Logger logger = LoggerFactory.getLogger(StockTickInput.class);
+ /**
+ * Timeout interval for reading from server. 0 or negative indicates no timeout.
+ */
+ public int readIntervalMillis = 500;
+ /**
+ * The URL of the web service resource for the POST request.
+ */
+ private String url;
+ private String[] symbols;
+ @NotNull
+ private String tickers;
+ private transient HttpClient client;
+ private transient GetMethod method;
+ private HashMap<String, Long> lastVolume = new HashMap<String, Long>();
+ private boolean outputEvenIfZeroVolume = false;
+ /**
+ * The output port to emit price.
+ */
+ @OutputPortFieldAnnotation(optional = true)
+ public final transient DefaultOutputPort<KeyValPair<String, Double>> price = new DefaultOutputPort<KeyValPair<String, Double>>();
+ /**
+ * The output port to emit incremental volume.
+ */
+ @OutputPortFieldAnnotation(optional = true)
+ public final transient DefaultOutputPort<KeyValPair<String, Long>> volume = new DefaultOutputPort<KeyValPair<String, Long>>();
+ /**
+ * The output port to emit last traded time.
+ */
+ @OutputPortFieldAnnotation(optional = true)
+ public final transient DefaultOutputPort<KeyValPair<String, String>> time = new DefaultOutputPort<KeyValPair<String, String>>();
+
+ /**
+ * Prepare URL from symbols and parameters. URL will be something like: http://download.finance.yahoo.com/d/quotes.csv?s=IBM,GOOG,AAPL,YHOO&f=sl1vt1
+ *
+ * @return the URL
+ */
+ private String prepareURL()
+ {
+ String str = "http://download.finance.yahoo.com/d/quotes.csv?s=";
+ for (int i = 0; i < symbols.length; i++) {
+ if (i != 0) {
+ str += ",";
+ }
+ str += symbols[i];
+ }
+ str += "&f=sl1vt1&e=.csv";
+ return str;
+ }
+
+ @Override
+ public void setup(OperatorContext context)
+ {
+ url = prepareURL();
+ client = new HttpClient();
+ method = new GetMethod(url);
+ DefaultHttpParams.getDefaultParams().setParameter("http.protocol.cookie-policy", CookiePolicy.BROWSER_COMPATIBILITY);
+ }
+
+ @Override
+ public void teardown()
+ {
+ }
+
+ @Override
+ public void emitTuples()
+ {
+
+ try {
+ int statusCode = client.executeMethod(method);
+ if (statusCode != HttpStatus.SC_OK) {
+ logger.error("Method failed: " + method.getStatusLine());
+ } else {
+ InputStream istream = method.getResponseBodyAsStream();
+ // Process response
+ InputStreamReader isr = new InputStreamReader(istream);
+ CSVReader reader = new CSVReader(isr);
+ List<String[]> myEntries = reader.readAll();
+ for (String[] stringArr: myEntries) {
+ ArrayList<String> tuple = new ArrayList<String>(Arrays.asList(stringArr));
+ if (tuple.size() != 4) {
+ return;
+ }
+ // input csv is <Symbol>,<Price>,<Volume>,<Time>
+ String symbol = tuple.get(0);
+ double currentPrice = Double.valueOf(tuple.get(1));
+ long currentVolume = Long.valueOf(tuple.get(2));
+ String timeStamp = tuple.get(3);
+ long vol = currentVolume;
+ // Sends total volume in first tick, and incremental volume afterwards.
+ if (lastVolume.containsKey(symbol)) {
+ vol -= lastVolume.get(symbol);
+ }
+
+ if (vol > 0 || outputEvenIfZeroVolume) {
+ price.emit(new KeyValPair<String, Double>(symbol, currentPrice));
+ volume.emit(new KeyValPair<String, Long>(symbol, vol));
+ time.emit(new KeyValPair<String, String>(symbol, timeStamp));
+ lastVolume.put(symbol, currentVolume);
+ }
+ }
+ }
+ Thread.sleep(readIntervalMillis);
+ } catch (InterruptedException ex) {
+ logger.debug(ex.toString());
+ } catch (IOException ex) {
+ logger.debug(ex.toString());
+ }
+ }
+
+ @Override
+ public void beginWindow(long windowId)
+ {
+ }
+
+ @Override
+ public void endWindow()
+ {
+ }
+
+ public void setOutputEvenIfZeroVolume(boolean outputEvenIfZeroVolume)
+ {
+ this.outputEvenIfZeroVolume = outputEvenIfZeroVolume;
+ }
+
+ public void setTickers(String tickers)
+ {
+ this.tickers = tickers;
+ symbols = StringUtils.split(tickers, ',');
+ }
+
+ public String getTickers()
+ {
+ return tickers;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/yahoofinance/src/main/java/org/apache/apex/examples/yahoofinance/YahooFinanceApplication.java
----------------------------------------------------------------------
diff --git a/examples/yahoofinance/src/main/java/org/apache/apex/examples/yahoofinance/YahooFinanceApplication.java b/examples/yahoofinance/src/main/java/org/apache/apex/examples/yahoofinance/YahooFinanceApplication.java
new file mode 100644
index 0000000..4cc323e
--- /dev/null
+++ b/examples/yahoofinance/src/main/java/org/apache/apex/examples/yahoofinance/YahooFinanceApplication.java
@@ -0,0 +1,365 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.yahoofinance;
+
+import org.apache.hadoop.conf.Configuration;
+import com.datatorrent.api.Context.OperatorContext;
+import com.datatorrent.api.Context.PortContext;
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.Operator.InputPort;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.lib.io.ConsoleOutputOperator;
+import com.datatorrent.lib.math.RangeKeyVal;
+import com.datatorrent.lib.math.SumKeyVal;
+import com.datatorrent.lib.multiwindow.SimpleMovingAverage;
+import com.datatorrent.lib.stream.ConsolidatorKeyVal;
+import com.datatorrent.lib.util.BaseKeyValueOperator.DefaultPartitionCodec;
+import com.datatorrent.lib.util.HighLow;
+
+/**
+ * Yahoo! Finance Application Example :<br>
+ * Get Yahoo finance feed and calculate minute price range, minute volume,
+ * simple moving average of 5 minutes. <br>
+ * <br>
+ * Functional Description : <br>
+ * Application samples yahoo finance ticker every 200ms. All data points in one
+ * second are streamed from input adapter. <br>
+ * <br>
+ *
+ * Application calculates following Real Time Value(s):<br>
+ * <ul>
+ * <li>Quotes for IBM, Google, Apple, Yahoo stocks price/volume/time displayed
+ * every second.</li>
+ * <li>Charts for Stocks in terms for high/low price vs volume for last minute.</li>
+ * <li>Simple moving average over last 5 minutes for IBM, Google, Apple, Yahoo
+ * stocks.</li>
+ * </ul>
+ * <br>
+ * <br>
+ *
+ * Custom Attribute : <br>
+ * <ul>
+ * <li>Application streaming window size(STREAMING_WINDOW_SIZE_MILLIS) = 1 sec,
+ * since we are only interested in quotes every second.</li>
+ * <li>Range/Minute Volume operator's window size(APPLICATION_WINDOW_COUNT) =
+ * 60, aggregate over one minute.</li>
+ * <li>Sum operator window length : 300, sliding average over last 5 minutes.</li>
+ * </ul>
+ * <br>
+ *
+ * Input Adapter : <br>
+ * Stock Tick input operator get yahoo finance real time stock quotes data and
+ * pushes application. <br>
+ * <br>
+ *
+ * Output Adapter : <br>
+ * Output values are written to console through ConsoleOutputOerator<br>
+ * if you need to change write to HDFS,HTTP .. instead of console, <br>
+ * Please refer to {@link com.datatorrent.lib.io.HttpOutputOperator} or
+ * {@link com.datatorrent.lib.io.fs.HdfsOutputOperator}. <br>
+ * <br>
+ *
+ * Run Sample Application : <br>
+ * <p>
+ * Running Java Test or Main app in IDE:
+ *
+ * <pre>
+ * LocalMode.runApp(new Application(), 600000); // 10 min run
+ * </pre>
+ *
+ * Run Success : <br>
+ * For successful deployment and run, user should see following output on
+ * console:
+ *
+ * <pre>
+ * Price SMA: AAPL=435.965
+ * Price SMA: GOOG=877.0
+ * QUOTE: {YHOO=[26.37, 9760360, 4:00pm, null, null], IBM=[203.77, 2899698, 4:00pm, null, null], GOOG=[877.0, 2069614, 4:00pm, null, null], AAPL=[435.965, 10208099, 4:00pm, null, null]}
+ * Price SMA: YHOO=26.37
+ * </pre>
+ *
+ * Scaling Options : <br>
+ * <ul>
+ * <li>Volume operator can be replicated using a {@link StatelessPartitioner}
+ * on an operator.</li>
+ * <li>Range value operator can replicated but using proper unifier
+ * operator(read App Dev Guide).</li>
+ * <li>Slinging window operator can be replicated with proper unifier operator.</li>
+ * </ul>
+ * <br>
+ *
+ * Application DAG : <br>
+ * <img src="doc-files/Application.gif" width=600px > <br>
+ * <br>
+ *
+ * Streaming Window Size : 1000 ms(1 Sec) <br>
+ * Operator Details : <br>
+ * <ul>
+ * <li>
+ * <p>
+ * <b>The operator DailyVolume:</b> This operator reads from the input port,
+ * which contains the incremental volume tuples from StockTickInput, and
+ * aggregates the data to provide the cumulative volume. It just utilizes the
+ * library class SumKeyVal<K,V> provided in math package. In this case,
+ * SumKeyVal<String,Long>, where K is the stock symbol, V is the aggregated
+ * volume, with cumulative set to true. (Otherwise if cumulative was set to
+ * false, SumKeyVal would provide the sum for the application window.) The platform
+ * provides a number of built-in operators for simple operations like this so
+ * that application developers do not have to write them. More examples to
+ * follow. This operator assumes that the application restarts before market
+ * opens every day.
+ * </p>
+ * Class : {@link com.datatorrent.lib.math.SumKeyVal} <br>
+ * Operator Application Window Count : 1 <br>
+ * StateFull : Yes, volume gets aggregated every window count.</li>
+ *
+ * <li>
+ * <p>
+ * <b>The operator MinuteVolume:</b> This operator reads from the input port,
+ * which contains the volume tuples from StockTickInput, and aggregates the data
+ * to provide the sum of the volume within one minute. Like the operator
+ * DailyVolume, this operator is also SumKeyVal<String,Long>, but with
+ * cumulative set to false. Application Window is set to 1 minute. We will
+ * explain how to set this later. <br>
+ * Class : {@link com.datatorrent.lib.math.SumKeyVal} <br>
+ * Operator App Window Count : 60 (1 Minute) <br>
+ * StateFull : Yes, aggregate over last 60 windows.</li>
+ *
+ * <li>
+ * <p>
+ * <b>The operator Quote:</b> This operator has three input ports, which are
+ * price (from StockTickInput), daily_vol (from Daily Volume), and time (from
+ * StockTickInput). This operator just consolidates the three data and and emits
+ * the consolidated data. It utilizes the class ConsolidatorKeyVal<K> from
+ * stream package.<br>
+ * Class : {@link com.datatorrent.lib.stream.ConsolidatorKeyVal} <br>
+ * Operator App Window Count : 1 <br>
+ * StateFull : No</li>
+ *
+ * <li>
+ * <p>
+ * <b>The operator Chart:</b> This operator is very similar to the operator
+ * Quote, except that it takes inputs from High Low and Minute Vol and outputs
+ * the consolidated tuples to the output port. <br>
+ * Class : {@link com.datatorrent.lib.stream.ConsolidatorKeyVal} <br>
+ * StateFull : No<br>
+ * Operator App Window Count : 1</li>
+ *
+ *
+ * <li>
+ * <p>
+ * <b>The operator PriceSMA:</b> SMA stands for - Simple Moving Average. It
+ * reads from the input port, which contains the price tuples from
+ * StockTickInput, and provides the moving average price of the stock. It
+ * utilizes SimpleMovingAverage<String,Double>, which is provided in multiwindow
+ * package. SimpleMovingAverage keeps track of the data of the previous N
+ * application windows in a sliding manner. For each end window event, it
+ * provides the average of the data in those application windows. <br>
+ * Class : {@link com.datatorrent.lib.multiwindow.SimpleMovingAverage} <br>
+ * StateFull : Yes, stores values across application window. <br>
+ * Operator App Window : 1 <br>
+ * Operator Sliding Window : 300 (5 mins).</li>
+ *
+ * <li>
+ * <p>
+ * <b>The operator Console: </b> This operator just outputs the input tuples to
+ * the console (or stdout). In this example, there are four console operators,
+ * which connect to the output of Quote, Chart, PriceSMA and VolumeSMA. In
+ * practice, they should be replaced by operators which would do something about
+ * the data, like drawing charts. </li>
+ *
+ * </ul>
+ * <br>
+ *
+ * @since 0.3.2
+ */
+@ApplicationAnnotation(name = "YahooFinanceExample")
+public class YahooFinanceApplication implements StreamingApplication
+{
+ protected int streamingWindowSizeMilliSeconds = 1000; // 1 second
+ protected int appWindowCountMinute = 60; // 1 minute
+ protected int appWindowCountSMA = 300; // 5 minute
+ //protected String[] tickers = {"IBM", "GOOG", "AAPL", "YHOO"};
+
+ /**
+ * Instantiate stock input operator for actual Yahoo finance ticks of symbol, last price, total daily volume, and last traded price.
+ * @param name Operator name
+ * @param dag Application DAG graph.
+ * @return StockTickInput instance.
+ */
+ public StockTickInput getStockTickInputOperator(String name, DAG dag)
+ {
+ StockTickInput oper = dag.addOperator(name, StockTickInput.class);
+ oper.readIntervalMillis = 200;
+ //oper.symbols = tickers;
+ return oper;
+ }
+
+ /**
+ * Instantiate {@link com.datatorrent.lib.math.SumKeyVal} operator
+ * to sends total daily volume by adding volumes from each ticks.
+ * @param name Operator name
+ * @param dag Application DAG graph.
+ * @return SumKeyVal instance.
+ */
+ public SumKeyVal<String, Long> getDailyVolumeOperator(String name, DAG dag)
+ {
+ SumKeyVal<String, Long> oper = dag.addOperator(name, new SumKeyVal<String, Long>());
+ oper.setType(Long.class);
+ oper.setCumulative(true);
+ return oper;
+ }
+
+ /**
+ * Instantiate {@link com.datatorrent.lib.math.SumKeyVal} operator
+ * Get aggregated volume of 1 minute and send at the end window of 1 minute.
+ * @param name Operator name
+ * @param dag Application DAG graph.
+ * @param appWindowCount Operator aggregate window size.
+ * @return SumKeyVal instance.
+ */
+ public SumKeyVal<String, Long> getMinuteVolumeOperator(String name, DAG dag, int appWindowCount)
+ {
+ SumKeyVal<String, Long> oper = dag.addOperator(name, new SumKeyVal<String, Long>());
+ oper.setType(Long.class);
+ dag.getOperatorMeta(name).getAttributes().put(OperatorContext.APPLICATION_WINDOW_COUNT, appWindowCount);
+ return oper;
+ }
+
+ /**
+ * Instantiate {@link com.datatorrent.lib.math.RangeKeyVal} operator to get high/low
+ * value for each key within given application window.
+ * Get High-low range for 1 minute.
+ * @param name Operator name
+ * @param dag Application DAG graph.
+ * @param appWindowCount Operator aggregate window size.
+ * @return RangeKeyVal instance.
+ */
+ public RangeKeyVal<String, Double> getHighLowOperator(String name, DAG dag, int appWindowCount)
+ {
+ RangeKeyVal<String, Double> oper = dag.addOperator(name, new RangeKeyVal<String, Double>());
+ dag.getOperatorMeta(name).getAttributes().put(OperatorContext.APPLICATION_WINDOW_COUNT, appWindowCount);
+ oper.setType(Double.class);
+ return oper;
+ }
+
+ /**
+ * Instantiate {@link com.datatorrent.lib.stream.ConsolidatorKeyVal} to send
+ * Quote (Merge price, daily volume, time)
+ * @param name Operator name
+ * @param dag Application DAG graph.
+ * @return ConsolidatorKeyVal instance.
+ */
+ public ConsolidatorKeyVal<String,Double,Long,String,?,?> getQuoteOperator(String name, DAG dag)
+ {
+ ConsolidatorKeyVal<String,Double,Long,String,?,?> oper = dag.addOperator(name, new ConsolidatorKeyVal<String,Double,Long,String,Object,Object>());
+ return oper;
+ }
+
+ /**
+ * Instantiate {@link com.datatorrent.lib.stream.ConsolidatorKeyVal} to send
+ * Chart (Merge minute volume and minute high-low)
+ * @param name Operator name
+ * @param dag Application DAG graph.
+ * @return ConsolidatorKeyVal instance.
+ */
+ public ConsolidatorKeyVal<String,HighLow<Double>,Long,?,?,?> getChartOperator(String name, DAG dag)
+ {
+ ConsolidatorKeyVal<String,HighLow<Double>,Long,?,?,?> oper = dag.addOperator(name, new ConsolidatorKeyVal<String,HighLow<Double>,Long,Object,Object,Object>());
+ return oper;
+ }
+
+ /**
+ * Instantiate {@link com.datatorrent.lib.multiwindow.SimpleMovingAverage} to calculate moving average for price
+ * over given window size. Sliding window size is 1.
+ * @param name Operator name
+ * @param dag Application DAG graph.
+ * @param appWindowCount Operator aggregate window size.
+ * @return SimpleMovingAverage instance.
+ */
+ public SimpleMovingAverage<String, Double> getPriceSimpleMovingAverageOperator(String name, DAG dag, int appWindowCount)
+ {
+ SimpleMovingAverage<String, Double> oper = dag.addOperator(name, new SimpleMovingAverage<String, Double>());
+ oper.setWindowSize(appWindowCount);
+ oper.setType(Double.class);
+ return oper;
+ }
+
+ /**
+ * Get console for output operator.
+ * @param name Operator name
+ * @param dag Application DAG graph.
+ * @return input port for console output.
+ */
+ public InputPort<Object> getConsole(String name, /*String nodeName,*/ DAG dag, String prefix)
+ {
+ // hack to output to HTTP based on actual environment
+ /*
+ String serverAddr = System.getenv("MALHAR_AJAXSERVER_ADDRESS");
+ if (serverAddr != null) {
+ HttpOutputOperator<Object> oper = dag.addOperator(name, new HttpOutputOperator<Object>());
+ oper.setResourceURL(URI.create("http://" + serverAddr + "/channel/" + nodeName));
+ return oper.input;
+ }
+ */
+
+ ConsoleOutputOperator oper = dag.addOperator(name, ConsoleOutputOperator.class);
+ oper.setStringFormat(prefix + ": %s");
+ return oper.input;
+ }
+
+ /**
+ * Populate Yahoo Finance Example Application DAG.
+ */
+ @SuppressWarnings("unchecked")
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+
+ dag.getAttributes().put(DAG.STREAMING_WINDOW_SIZE_MILLIS, streamingWindowSizeMilliSeconds);
+
+ StockTickInput tick = getStockTickInputOperator("StockTickInput", dag);
+ SumKeyVal<String, Long> dailyVolume = getDailyVolumeOperator("DailyVolume", dag);
+ ConsolidatorKeyVal<String,Double,Long,String,?,?> quoteOperator = getQuoteOperator("Quote", dag);
+
+ RangeKeyVal<String, Double> highlow = getHighLowOperator("HighLow", dag, appWindowCountMinute);
+ SumKeyVal<String, Long> minuteVolume = getMinuteVolumeOperator("MinuteVolume", dag, appWindowCountMinute);
+ ConsolidatorKeyVal<String,HighLow<Double>,Long,?,?,?> chartOperator = getChartOperator("Chart", dag);
+
+ SimpleMovingAverage<String, Double> priceSMA = getPriceSimpleMovingAverageOperator("PriceSMA", dag, appWindowCountSMA);
+ DefaultPartitionCodec<String, Double> codec = new DefaultPartitionCodec<String, Double>();
+ dag.setInputPortAttribute(highlow.data, PortContext.STREAM_CODEC, codec);
+ dag.setInputPortAttribute(priceSMA.data, PortContext.STREAM_CODEC, codec);
+ dag.addStream("price", tick.price, quoteOperator.in1, highlow.data, priceSMA.data);
+ dag.addStream("vol", tick.volume, dailyVolume.data, minuteVolume.data);
+ dag.addStream("time", tick.time, quoteOperator.in3);
+ dag.addStream("daily_vol", dailyVolume.sum, quoteOperator.in2);
+
+ dag.addStream("quote_data", quoteOperator.out, getConsole("quoteConsole", dag, "QUOTE"));
+
+ dag.addStream("high_low", highlow.range, chartOperator.in1);
+ dag.addStream("vol_1min", minuteVolume.sum, chartOperator.in2);
+ dag.addStream("chart_data", chartOperator.out, getConsole("chartConsole", dag, "CHART"));
+
+ dag.addStream("sma_price", priceSMA.doubleSMA, getConsole("priceSMAConsole", dag, "Price SMA"));
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/yahoofinance/src/main/java/org/apache/apex/examples/yahoofinance/YahooFinanceCSVInputOperator.java
----------------------------------------------------------------------
diff --git a/examples/yahoofinance/src/main/java/org/apache/apex/examples/yahoofinance/YahooFinanceCSVInputOperator.java b/examples/yahoofinance/src/main/java/org/apache/apex/examples/yahoofinance/YahooFinanceCSVInputOperator.java
new file mode 100644
index 0000000..86b6263
--- /dev/null
+++ b/examples/yahoofinance/src/main/java/org/apache/apex/examples/yahoofinance/YahooFinanceCSVInputOperator.java
@@ -0,0 +1,160 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.yahoofinance;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.httpclient.HttpClient;
+import org.apache.commons.httpclient.HttpStatus;
+import org.apache.commons.httpclient.cookie.CookiePolicy;
+import org.apache.commons.httpclient.methods.GetMethod;
+import org.apache.commons.httpclient.params.DefaultHttpParams;
+
+import com.datatorrent.api.Context.OperatorContext;
+import com.datatorrent.lib.io.SimpleSinglePortInputOperator;
+
+import au.com.bytecode.opencsv.CSVReader;
+
+/**
+ * Grabs Yahoo Finance quotes data and emits HashMap, with key equals the format name (e.g. "s0") <p>
+ *
+ * @since 0.3.2
+ */
+public class YahooFinanceCSVInputOperator extends SimpleSinglePortInputOperator<HashMap<String, Object>> implements Runnable
+{
+ private static final Logger logger = LoggerFactory.getLogger(YahooFinanceCSVInputOperator.class);
+ /**
+ * Timeout interval for reading from server. 0 or negative indicates no timeout.
+ */
+ private int readIntervalMillis = 500;
+
+ /**
+ * The URL of the web service resource for the POST request.
+ */
+ private String url;
+ private transient HttpClient client;
+ private transient GetMethod method;
+
+ private ArrayList<String> symbolList = new ArrayList<String>();
+ private ArrayList<String> parameterList = new ArrayList<String>();
+
+ public void addSymbol(String symbol)
+ {
+ symbolList.add(symbol);
+ }
+
+ public void addFormat(String format)
+ {
+ parameterList.add(format);
+ }
+
+ public ArrayList<String> getSymbolList()
+ {
+ return symbolList;
+ }
+
+ public ArrayList<String> getParameterList()
+ {
+ return parameterList;
+ }
+
+ public int getReadIntervalMillis()
+ {
+ return readIntervalMillis;
+ }
+
+ public void setReadIntervalMillis(int readIntervalMillis)
+ {
+ this.readIntervalMillis = readIntervalMillis;
+ }
+
+ /**
+ * Prepare URL from symbols and parameters.
+ * URL will be something like: http://download.finance.yahoo.com/d/quotes.csv?s=GOOG,FB,YHOO&f=sl1vt1&e=.csv
+ * @return
+ */
+ private String prepareURL()
+ {
+ String str = "http://download.finance.yahoo.com/d/quotes.csv?";
+
+ str += "s=";
+ for (int i = 0; i < symbolList.size(); i++) {
+ if (i != 0) {
+ str += ",";
+ }
+ str += symbolList.get(i);
+ }
+ str += "&f=";
+ for (String format: parameterList) {
+ str += format;
+ }
+ str += "&e=.csv";
+ return str;
+ }
+
+ @Override
+ public void setup(OperatorContext context)
+ {
+ url = prepareURL();
+ client = new HttpClient();
+ method = new GetMethod(url);
+ DefaultHttpParams.getDefaultParams().setParameter("http.protocol.cookie-policy", CookiePolicy.BROWSER_COMPATIBILITY);
+ }
+
+ @Override
+ public void run()
+ {
+ while (true) {
+ try {
+ int statusCode = client.executeMethod(method);
+ if (statusCode != HttpStatus.SC_OK) {
+ logger.error("Method failed: " + method.getStatusLine());
+ } else {
+ InputStream istream;
+ istream = method.getResponseBodyAsStream();
+ // Process response
+ InputStreamReader isr = new InputStreamReader(istream);
+ CSVReader reader = new CSVReader(isr);
+ List<String[]> myEntries;
+ myEntries = reader.readAll();
+ for (String[] stringArr: myEntries) {
+ HashMap<String,Object> hm = new HashMap<String,Object>();
+ for (int i = 0; i < parameterList.size(); i++) {
+ hm.put(parameterList.get(i), stringArr[i]);
+ }
+ outputPort.emit(hm); // send out one symbol at a time
+ }
+ }
+ Thread.sleep(readIntervalMillis);
+ } catch (InterruptedException ex) {
+ logger.debug(ex.toString());
+ } catch (IOException ex) {
+ logger.debug(ex.toString());
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/yahoofinance/src/main/java/org/apache/apex/examples/yahoofinance/doc-files/Application.gif
----------------------------------------------------------------------
diff --git a/examples/yahoofinance/src/main/java/org/apache/apex/examples/yahoofinance/doc-files/Application.gif b/examples/yahoofinance/src/main/java/org/apache/apex/examples/yahoofinance/doc-files/Application.gif
new file mode 100644
index 0000000..8fc8331
Binary files /dev/null and b/examples/yahoofinance/src/main/java/org/apache/apex/examples/yahoofinance/doc-files/Application.gif differ
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/yahoofinance/src/main/java/org/apache/apex/examples/yahoofinance/package-info.java
----------------------------------------------------------------------
diff --git a/examples/yahoofinance/src/main/java/org/apache/apex/examples/yahoofinance/package-info.java b/examples/yahoofinance/src/main/java/org/apache/apex/examples/yahoofinance/package-info.java
new file mode 100644
index 0000000..4a7bb10
--- /dev/null
+++ b/examples/yahoofinance/src/main/java/org/apache/apex/examples/yahoofinance/package-info.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/**
+ * Yahoo Finance demonstration applications.
+ */
+package org.apache.apex.examples.yahoofinance;
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/yahoofinance/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/examples/yahoofinance/src/main/resources/META-INF/properties.xml b/examples/yahoofinance/src/main/resources/META-INF/properties.xml
new file mode 100644
index 0000000..bb5f201
--- /dev/null
+++ b/examples/yahoofinance/src/main/resources/META-INF/properties.xml
@@ -0,0 +1,81 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<!--Configuration for chart example -->
+<configuration>
+ <property>
+ <name>dt.attr.MASTER_MEMORY_MB</name>
+ <value>1024</value>
+ </property>
+ <property>
+ <name>dt.attr.STREAMING_WINDOW_SIZE_MILLIS</name>
+ <value>1000</value>
+ </property>
+ <property>
+ <name>dt.application.YahooFinanceExampleWithChart.operator.AverageChart.xAxisLabel</name>
+ <value>TIME</value>
+ </property>
+ <property>
+ <name>dt.application.YahooFinanceExampleWithChart.operator.StockTickInput.tickers</name>
+ <value>IBM,GOOG,AAPL,YHOO</value>
+ </property>
+ <property>
+ <name>dt.application.*.operator.*.attr.MEMORY_MB</name>
+ <value>256</value>
+ </property>
+ <property>
+ <name>dt.application.*.operator.*.attr.JVM_OPTIONS</name>
+ <value>-Xmx128M</value>
+ </property>
+ <property>
+ <name>dt.application.*.operator.*.port.*.attr.BUFFER_MEMORY_MB</name>
+ <value>256</value>
+ </property>
+ <property>
+ <name>dt.application.YahooFinanceExample.operator.StockTickInput.tickers</name>
+ <value>IBM,GOOG,AAPL,YHOO</value>
+ </property>
+ <property>
+ <name>dt.application.YahooFinanceWithoutChartExample.operator.StockTickInput.tickers</name>
+ <value>IBM,GOOG,AAPL,YHOO</value>
+ </property>
+ <property>
+ <name>dt.application.YahooFinanceExampleWithChart.operator.AverageChart.yAxisLabel</name>
+ <value>PRICE</value>
+ </property>
+ <property>
+ <name>dt.application.YahooFinanceExampleWithChart.operator.CandleStickChart.xAxisLabel</name>
+ <value>TIME</value>
+ </property>
+ <property>
+ <name>dt.application.YahooFinanceExampleWithChart.operator.CandleStickChart.yAxisLabel</name>
+ <value>PRICE</value>
+ </property>
+ <property>
+ <name>dt.application.YahooFinanceExampleWithChart.operator.AverageChart.attr.APPLICATION_WINDOW_COUNT
+ </name>
+ <value>5</value>
+ </property>
+ <property>
+ <name>dt.application.YahooFinanceExampleWithChart.operator.AverageChart.attr.APPLICATION_WINDOW_COUNT
+ </name>
+ <value>5</value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/yahoofinance/src/test/java/org/apache/apex/examples/yahoofinance/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/yahoofinance/src/test/java/org/apache/apex/examples/yahoofinance/ApplicationTest.java b/examples/yahoofinance/src/test/java/org/apache/apex/examples/yahoofinance/ApplicationTest.java
new file mode 100644
index 0000000..a8c3cb9
--- /dev/null
+++ b/examples/yahoofinance/src/test/java/org/apache/apex/examples/yahoofinance/ApplicationTest.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.yahoofinance;
+
+import org.junit.Test;
+import org.apache.hadoop.conf.Configuration;
+import com.datatorrent.api.LocalMode;
+
+/**
+ * Run Yahoo Finance application example.
+ *
+ */
+public class ApplicationTest
+{
+
+ /**
+ * This will run for ever.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testApplication() throws Exception
+ {
+ LocalMode lma = LocalMode.newInstance();
+ new YahooFinanceApplication().populateDAG(lma.getDAG(), new Configuration(false));
+ LocalMode.Controller lc = lma.getController();
+ lc.run(10000);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/yahoofinance/src/test/java/org/apache/apex/examples/yahoofinance/ApplicationWithDerbySQLTest.java
----------------------------------------------------------------------
diff --git a/examples/yahoofinance/src/test/java/org/apache/apex/examples/yahoofinance/ApplicationWithDerbySQLTest.java b/examples/yahoofinance/src/test/java/org/apache/apex/examples/yahoofinance/ApplicationWithDerbySQLTest.java
new file mode 100644
index 0000000..bd2db3d
--- /dev/null
+++ b/examples/yahoofinance/src/test/java/org/apache/apex/examples/yahoofinance/ApplicationWithDerbySQLTest.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.yahoofinance;
+
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.hadoop.conf.Configuration;
+import com.datatorrent.api.LocalMode;
+
+/**
+ *
+ */
+public class ApplicationWithDerbySQLTest
+{
+ private final transient Logger LOG = LoggerFactory.getLogger(ApplicationWithDerbySQLTest.class);
+ public ApplicationWithDerbySQLTest()
+ {
+ }
+
+ @Test
+ public void testSomeMethod() throws Exception
+ {
+ LocalMode lma = LocalMode.newInstance();
+ new ApplicationWithDerbySQL().populateDAG(lma.getDAG(), new Configuration(false));
+ LocalMode.Controller lc = lma.getController();
+
+ long start = System.currentTimeMillis();
+ lc.run();
+ long end = System.currentTimeMillis();
+ long time = end - start;
+ LOG.debug("Test used " + time + " ms");
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/yahoofinance/src/test/resources/alert_create.json
----------------------------------------------------------------------
diff --git a/examples/yahoofinance/src/test/resources/alert_create.json b/examples/yahoofinance/src/test/resources/alert_create.json
new file mode 100644
index 0000000..3059145
--- /dev/null
+++ b/examples/yahoofinance/src/test/resources/alert_create.json
@@ -0,0 +1,24 @@
+{
+ "name":"alertName",
+ "streamName":"yahooFinance.outputPort",
+ "filter": {
+ "class": "com.datatorrent.lib.util.JavaScriptFilterOperator",
+ "properties": {
+ "setupScript":"function f() { return s0 == \"AAPL\" && l1 > 508 }",
+ "functionName":"f"
+ }
+ },
+ "escalation": {
+ "class": "com.datatorrent.lib.util.AlertEscalationOperator",
+ "properties": {
+ "alertInterval":"5000",
+ "timeout":"10000"
+ }
+ },
+ "actions": [{
+ "outputPort":"alert",
+ "inputPort":"input",
+ "class":"com.datatorrent.lib.io.ConsoleOutputOperator"
+ }],
+ "saveAs":"firstAlert"
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/yahoofinance/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/examples/yahoofinance/src/test/resources/log4j.properties b/examples/yahoofinance/src/test/resources/log4j.properties
new file mode 100644
index 0000000..cf0d19e
--- /dev/null
+++ b/examples/yahoofinance/src/test/resources/log4j.properties
@@ -0,0 +1,43 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+log4j.rootLogger=DEBUG,CONSOLE
+
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+log4j.appender.CONSOLE.threshold=${test.log.console.threshold}
+test.log.console.threshold=DEBUG
+
+log4j.appender.RFA=org.apache.log4j.RollingFileAppender
+log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
+log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+log4j.appender.RFA.File=/tmp/app.log
+
+# to enable, add SYSLOG to rootLogger
+log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender
+log4j.appender.SYSLOG.syslogHost=127.0.0.1
+log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout
+log4j.appender.SYSLOG.layout.conversionPattern=${dt.cid} %-5p [%t] %c{2} %x - %m%n
+log4j.appender.SYSLOG.Facility=LOCAL1
+
+log4j.logger.org=info
+#log4j.logger.org.apache.commons.beanutils=warn
+log4j.logger.com.datatorrent=debug
+log4j.logger.org.apache.apex=debug
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/library/src/test/resources/SocketInputOperatorTest.txt
----------------------------------------------------------------------
diff --git a/library/src/test/resources/SocketInputOperatorTest.txt b/library/src/test/resources/SocketInputOperatorTest.txt
index 319c661..914cd5d 100644
--- a/library/src/test/resources/SocketInputOperatorTest.txt
+++ b/library/src/test/resources/SocketInputOperatorTest.txt
@@ -1 +1 @@
-Malhar repository contains open source operator and codec library that can be used with the Apache Apex (incubating) platform to build Realtime streaming applications. In addition to the library there are benchmark, contrib, demos, webdemos and samples folders available. Demos contain demo applications built using the library operators. Webdemos contain webpages for the demos. Benchmark contains performance testing applications. Contrib contains additional operators that interface with third party softwares. Samples contain some sample code that shows how to use the library operators.
+Malhar repository contains open source operator and codec library that can be used with the Apache Apex (incubating) platform to build Realtime streaming applications. In addition to the library there are benchmark, contrib, examples, webdemos and samples folders available. Examples contain example applications built using the library operators. Webdemos contain webpages for the examples. Benchmark contains performance testing applications. Contrib contains additional operators that interface with third party softwares. Samples contain some sample code that shows how to use the library operators.
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 79de8f4..9cc611a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -210,7 +210,7 @@
<modules>
<module>library</module>
<module>contrib</module>
- <module>demos</module>
+ <module>examples</module>
</modules>
<dependencies>
[09/30] apex-malhar git commit: Renamed demos to examples. Packages
and artifactid names are changed as suggested.
Posted by th...@apache.org.
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/machinedata/src/main/html/malhar.css
----------------------------------------------------------------------
diff --git a/examples/machinedata/src/main/html/malhar.css b/examples/machinedata/src/main/html/malhar.css
new file mode 100644
index 0000000..175e219
--- /dev/null
+++ b/examples/machinedata/src/main/html/malhar.css
@@ -0,0 +1,4688 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+.clearfix {
+ *zoom: 1;
+}
+.clearfix:before,
+.clearfix:after {
+ display: table;
+ content: "";
+ line-height: 0;
+}
+.clearfix:after {
+ clear: both;
+}
+.hide-text {
+ font: 0/0 a;
+ color: transparent;
+ text-shadow: none;
+ background-color: transparent;
+ border: 0;
+}
+.input-block-level {
+ display: block;
+ width: 100%;
+ min-height: 30px;
+ -webkit-box-sizing: border-box;
+ -moz-box-sizing: border-box;
+ box-sizing: border-box;
+}
+article,
+aside,
+details,
+figcaption,
+figure,
+footer,
+header,
+hgroup,
+nav,
+section {
+ display: block;
+}
+audio,
+canvas,
+video {
+ display: inline-block;
+ *display: inline;
+ *zoom: 1;
+}
+audio:not([controls]) {
+ display: none;
+}
+html {
+ font-size: 100%;
+ -webkit-text-size-adjust: 100%;
+ -ms-text-size-adjust: 100%;
+}
+a:focus {
+ outline: thin dotted #333;
+ outline: 5px auto -webkit-focus-ring-color;
+ outline-offset: -2px;
+}
+a:hover,
+a:active {
+ outline: 0;
+}
+sub,
+sup {
+ position: relative;
+ font-size: 75%;
+ line-height: 0;
+ vertical-align: baseline;
+}
+sup {
+ top: -0.5em;
+}
+sub {
+ bottom: -0.25em;
+}
+img {
+ /* Responsive images (ensure images don't scale beyond their parents) */
+
+ max-width: 100%;
+ /* Part 1: Set a maxium relative to the parent */
+
+ width: auto\9;
+ /* IE7-8 need help adjusting responsive images */
+
+ height: auto;
+ /* Part 2: Scale the height according to the width, otherwise you get stretching */
+
+ vertical-align: middle;
+ border: 0;
+ -ms-interpolation-mode: bicubic;
+}
+#map_canvas img,
+.google-maps img {
+ max-width: none;
+}
+button,
+input,
+select,
+textarea {
+ margin: 0;
+ font-size: 100%;
+ vertical-align: middle;
+}
+button,
+input {
+ *overflow: visible;
+ line-height: normal;
+}
+button::-moz-focus-inner,
+input::-moz-focus-inner {
+ padding: 0;
+ border: 0;
+}
+button,
+html input[type="button"],
+input[type="reset"],
+input[type="submit"] {
+ -webkit-appearance: button;
+ cursor: pointer;
+}
+label,
+select,
+button,
+input[type="button"],
+input[type="reset"],
+input[type="submit"],
+input[type="radio"],
+input[type="checkbox"] {
+ cursor: pointer;
+}
+input[type="search"] {
+ -webkit-box-sizing: content-box;
+ -moz-box-sizing: content-box;
+ box-sizing: content-box;
+ -webkit-appearance: textfield;
+}
+input[type="search"]::-webkit-search-decoration,
+input[type="search"]::-webkit-search-cancel-button {
+ -webkit-appearance: none;
+}
+textarea {
+ overflow: auto;
+ vertical-align: top;
+}
+@media print {
+ * {
+ text-shadow: none !important;
+ color: #000 !important;
+ background: transparent !important;
+ box-shadow: none !important;
+ }
+ a,
+ a:visited {
+ text-decoration: underline;
+ }
+ a[href]:after {
+ content: " (" attr(href) ")";
+ }
+ abbr[title]:after {
+ content: " (" attr(title) ")";
+ }
+ .ir a:after,
+ a[href^="javascript:"]:after,
+ a[href^="#"]:after {
+ content: "";
+ }
+ pre,
+ blockquote {
+ border: 1px solid #999;
+ page-break-inside: avoid;
+ }
+ thead {
+ display: table-header-group;
+ }
+ tr,
+ img {
+ page-break-inside: avoid;
+ }
+ img {
+ max-width: 100% !important;
+ }
+ @page {
+ margin: 0.5cm;
+ }
+ p,
+ h2,
+ h3 {
+ orphans: 3;
+ widows: 3;
+ }
+ h2,
+ h3 {
+ page-break-after: avoid;
+ }
+}
+body {
+ margin: 0;
+ font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
+ font-size: 13px;
+ line-height: 20px;
+ color: #333333;
+ background-color: #ffffff;
+}
+a {
+ color: #0088cc;
+ text-decoration: none;
+}
+a:hover,
+a:focus {
+ color: #005580;
+ text-decoration: underline;
+}
+.img-rounded {
+ -webkit-border-radius: 6px;
+ -moz-border-radius: 6px;
+ border-radius: 6px;
+}
+.img-polaroid {
+ padding: 4px;
+ background-color: #fff;
+ border: 1px solid #ccc;
+ border: 1px solid rgba(0, 0, 0, 0.2);
+ -webkit-box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1);
+ -moz-box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1);
+ box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1);
+}
+.img-circle {
+ -webkit-border-radius: 500px;
+ -moz-border-radius: 500px;
+ border-radius: 500px;
+}
+/*@import "bootstrap/grid.less";*/
+/*@import "bootstrap/layouts.less";*/
+p {
+ margin: 0 0 10px;
+}
+.lead {
+ margin-bottom: 20px;
+ font-size: 19.5px;
+ font-weight: 200;
+ line-height: 30px;
+}
+small {
+ font-size: 85%;
+}
+strong {
+ font-weight: bold;
+}
+em {
+ font-style: italic;
+}
+cite {
+ font-style: normal;
+}
+.muted {
+ color: #999999;
+}
+a.muted:hover,
+a.muted:focus {
+ color: #808080;
+}
+.text-warning {
+ color: #c09853;
+}
+a.text-warning:hover,
+a.text-warning:focus {
+ color: #a47e3c;
+}
+.text-error {
+ color: #b94a48;
+}
+a.text-error:hover,
+a.text-error:focus {
+ color: #953b39;
+}
+.text-info {
+ color: #3a87ad;
+}
+a.text-info:hover,
+a.text-info:focus {
+ color: #2d6987;
+}
+.text-success {
+ color: #468847;
+}
+a.text-success:hover,
+a.text-success:focus {
+ color: #356635;
+}
+.text-left {
+ text-align: left;
+}
+.text-right {
+ text-align: right;
+}
+.text-center {
+ text-align: center;
+}
+h1,
+h2,
+h3,
+h4,
+h5,
+h6 {
+ margin: 10px 0;
+ font-family: inherit;
+ font-weight: bold;
+ line-height: 20px;
+ color: inherit;
+ text-rendering: optimizelegibility;
+}
+h1 small,
+h2 small,
+h3 small,
+h4 small,
+h5 small,
+h6 small {
+ font-weight: normal;
+ line-height: 1;
+ color: #999999;
+}
+h1,
+h2,
+h3 {
+ line-height: 40px;
+}
+h1 {
+ font-size: 35.75px;
+}
+h2 {
+ font-size: 29.25px;
+}
+h3 {
+ font-size: 22.75px;
+}
+h4 {
+ font-size: 16.25px;
+}
+h5 {
+ font-size: 13px;
+}
+h6 {
+ font-size: 11.049999999999999px;
+}
+h1 small {
+ font-size: 22.75px;
+}
+h2 small {
+ font-size: 16.25px;
+}
+h3 small {
+ font-size: 13px;
+}
+h4 small {
+ font-size: 13px;
+}
+.page-header {
+ padding-bottom: 9px;
+ margin: 20px 0 30px;
+ border-bottom: 1px solid #eeeeee;
+}
+ul,
+ol {
+ padding: 0;
+ margin: 0 0 10px 25px;
+}
+ul ul,
+ul ol,
+ol ol,
+ol ul {
+ margin-bottom: 0;
+}
+li {
+ line-height: 20px;
+}
+ul.unstyled,
+ol.unstyled {
+ margin-left: 0;
+ list-style: none;
+}
+ul.inline,
+ol.inline {
+ margin-left: 0;
+ list-style: none;
+}
+ul.inline > li,
+ol.inline > li {
+ display: inline-block;
+ *display: inline;
+ /* IE7 inline-block hack */
+
+ *zoom: 1;
+ padding-left: 5px;
+ padding-right: 5px;
+}
+dl {
+ margin-bottom: 20px;
+}
+dt,
+dd {
+ line-height: 20px;
+}
+dt {
+ font-weight: bold;
+}
+dd {
+ margin-left: 10px;
+}
+.dl-horizontal {
+ *zoom: 1;
+}
+.dl-horizontal:before,
+.dl-horizontal:after {
+ display: table;
+ content: "";
+ line-height: 0;
+}
+.dl-horizontal:after {
+ clear: both;
+}
+.dl-horizontal dt {
+ float: left;
+ width: 160px;
+ clear: left;
+ text-align: right;
+ overflow: hidden;
+ text-overflow: ellipsis;
+ white-space: nowrap;
+}
+.dl-horizontal dd {
+ margin-left: 180px;
+}
+hr {
+ margin: 20px 0;
+ border: 0;
+ border-top: 1px solid #eeeeee;
+ border-bottom: 1px solid #ffffff;
+}
+abbr[title],
+abbr[data-original-title] {
+ cursor: help;
+ border-bottom: 1px dotted #999999;
+}
+abbr.initialism {
+ font-size: 90%;
+ text-transform: uppercase;
+}
+blockquote {
+ padding: 0 0 0 15px;
+ margin: 0 0 20px;
+ border-left: 5px solid #eeeeee;
+}
+blockquote p {
+ margin-bottom: 0;
+ font-size: 16.25px;
+ font-weight: 300;
+ line-height: 1.25;
+}
+blockquote small {
+ display: block;
+ line-height: 20px;
+ color: #999999;
+}
+blockquote small:before {
+ content: '\2014 \00A0';
+}
+blockquote.pull-right {
+ float: right;
+ padding-right: 15px;
+ padding-left: 0;
+ border-right: 5px solid #eeeeee;
+ border-left: 0;
+}
+blockquote.pull-right p,
+blockquote.pull-right small {
+ text-align: right;
+}
+blockquote.pull-right small:before {
+ content: '';
+}
+blockquote.pull-right small:after {
+ content: '\00A0 \2014';
+}
+q:before,
+q:after,
+blockquote:before,
+blockquote:after {
+ content: "";
+}
+address {
+ display: block;
+ margin-bottom: 20px;
+ font-style: normal;
+ line-height: 20px;
+}
+code,
+pre {
+ padding: 0 3px 2px;
+ font-family: Monaco, Menlo, Consolas, "Courier New", monospace;
+ font-size: 11px;
+ color: #333333;
+ -webkit-border-radius: 3px;
+ -moz-border-radius: 3px;
+ border-radius: 3px;
+}
+code {
+ padding: 2px 4px;
+ color: #d14;
+ background-color: #f7f7f9;
+ border: 1px solid #e1e1e8;
+ white-space: nowrap;
+}
+pre {
+ display: block;
+ padding: 9.5px;
+ margin: 0 0 10px;
+ font-size: 12px;
+ line-height: 20px;
+ word-break: break-all;
+ word-wrap: break-word;
+ white-space: pre;
+ white-space: pre-wrap;
+ background-color: #f5f5f5;
+ border: 1px solid #ccc;
+ border: 1px solid rgba(0, 0, 0, 0.15);
+ -webkit-border-radius: 4px;
+ -moz-border-radius: 4px;
+ border-radius: 4px;
+}
+pre.prettyprint {
+ margin-bottom: 20px;
+}
+pre code {
+ padding: 0;
+ color: inherit;
+ white-space: pre;
+ white-space: pre-wrap;
+ background-color: transparent;
+ border: 0;
+}
+.pre-scrollable {
+ max-height: 340px;
+ overflow-y: scroll;
+}
+form {
+ margin: 0 0 20px;
+}
+fieldset {
+ padding: 0;
+ margin: 0;
+ border: 0;
+}
+legend {
+ display: block;
+ width: 100%;
+ padding: 0;
+ margin-bottom: 20px;
+ font-size: 19.5px;
+ line-height: 40px;
+ color: #333333;
+ border: 0;
+ border-bottom: 1px solid #e5e5e5;
+}
+legend small {
+ font-size: 15px;
+ color: #999999;
+}
+label,
+input,
+button,
+select,
+textarea {
+ font-size: 13px;
+ font-weight: normal;
+ line-height: 20px;
+}
+input,
+button,
+select,
+textarea {
+ font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
+}
+label {
+ display: block;
+ margin-bottom: 5px;
+}
+select,
+textarea,
+input[type="text"],
+input[type="password"],
+input[type="datetime"],
+input[type="datetime-local"],
+input[type="date"],
+input[type="month"],
+input[type="time"],
+input[type="week"],
+input[type="number"],
+input[type="email"],
+input[type="url"],
+input[type="search"],
+input[type="tel"],
+input[type="color"],
+.uneditable-input {
+ display: inline-block;
+ height: 20px;
+ padding: 4px 6px;
+ margin-bottom: 10px;
+ font-size: 13px;
+ line-height: 20px;
+ color: #555555;
+ -webkit-border-radius: 4px;
+ -moz-border-radius: 4px;
+ border-radius: 4px;
+ vertical-align: middle;
+}
+input,
+textarea,
+.uneditable-input {
+ width: 206px;
+}
+textarea {
+ height: auto;
+}
+textarea,
+input[type="text"],
+input[type="password"],
+input[type="datetime"],
+input[type="datetime-local"],
+input[type="date"],
+input[type="month"],
+input[type="time"],
+input[type="week"],
+input[type="number"],
+input[type="email"],
+input[type="url"],
+input[type="search"],
+input[type="tel"],
+input[type="color"],
+.uneditable-input {
+ background-color: #ffffff;
+ border: 1px solid #cccccc;
+ -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
+ -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
+ box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
+ -webkit-transition: border linear .2s, box-shadow linear .2s;
+ -moz-transition: border linear .2s, box-shadow linear .2s;
+ -o-transition: border linear .2s, box-shadow linear .2s;
+ transition: border linear .2s, box-shadow linear .2s;
+}
+textarea:focus,
+input[type="text"]:focus,
+input[type="password"]:focus,
+input[type="datetime"]:focus,
+input[type="datetime-local"]:focus,
+input[type="date"]:focus,
+input[type="month"]:focus,
+input[type="time"]:focus,
+input[type="week"]:focus,
+input[type="number"]:focus,
+input[type="email"]:focus,
+input[type="url"]:focus,
+input[type="search"]:focus,
+input[type="tel"]:focus,
+input[type="color"]:focus,
+.uneditable-input:focus {
+ border-color: rgba(82, 168, 236, 0.8);
+ outline: 0;
+ outline: thin dotted \9;
+ /* IE6-9 */
+
+ -webkit-box-shadow: inset 0 1px 1px rgba(0,0,0,.075), 0 0 8px rgba(82,168,236,.6);
+ -moz-box-shadow: inset 0 1px 1px rgba(0,0,0,.075), 0 0 8px rgba(82,168,236,.6);
+ box-shadow: inset 0 1px 1px rgba(0,0,0,.075), 0 0 8px rgba(82,168,236,.6);
+}
+input[type="radio"],
+input[type="checkbox"] {
+ margin: 4px 0 0;
+ *margin-top: 0;
+ /* IE7 */
+
+ margin-top: 1px \9;
+ /* IE8-9 */
+
+ line-height: normal;
+}
+input[type="file"],
+input[type="image"],
+input[type="submit"],
+input[type="reset"],
+input[type="button"],
+input[type="radio"],
+input[type="checkbox"] {
+ width: auto;
+}
+select,
+input[type="file"] {
+ height: 30px;
+ /* In IE7, the height of the select element cannot be changed by height, only font-size */
+
+ *margin-top: 4px;
+ /* For IE7, add top margin to align select with labels */
+
+ line-height: 30px;
+}
+select {
+ width: 220px;
+ border: 1px solid #cccccc;
+ background-color: #ffffff;
+}
+select[multiple],
+select[size] {
+ height: auto;
+}
+select:focus,
+input[type="file"]:focus,
+input[type="radio"]:focus,
+input[type="checkbox"]:focus {
+ outline: thin dotted #333;
+ outline: 5px auto -webkit-focus-ring-color;
+ outline-offset: -2px;
+}
+.uneditable-input,
+.uneditable-textarea {
+ color: #999999;
+ background-color: #fcfcfc;
+ border-color: #cccccc;
+ -webkit-box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.025);
+ -moz-box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.025);
+ box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.025);
+ cursor: not-allowed;
+}
+.uneditable-input {
+ overflow: hidden;
+ white-space: nowrap;
+}
+.uneditable-textarea {
+ width: auto;
+ height: auto;
+}
+input:-moz-placeholder,
+textarea:-moz-placeholder {
+ color: #999999;
+}
+input:-ms-input-placeholder,
+textarea:-ms-input-placeholder {
+ color: #999999;
+}
+input::-webkit-input-placeholder,
+textarea::-webkit-input-placeholder {
+ color: #999999;
+}
+.radio,
+.checkbox {
+ min-height: 20px;
+ padding-left: 20px;
+}
+.radio input[type="radio"],
+.checkbox input[type="checkbox"] {
+ float: left;
+ margin-left: -20px;
+}
+.controls > .radio:first-child,
+.controls > .checkbox:first-child {
+ padding-top: 5px;
+}
+.radio.inline,
+.checkbox.inline {
+ display: inline-block;
+ padding-top: 5px;
+ margin-bottom: 0;
+ vertical-align: middle;
+}
+.radio.inline + .radio.inline,
+.checkbox.inline + .checkbox.inline {
+ margin-left: 10px;
+}
+.input-mini {
+ width: 60px;
+}
+.input-small {
+ width: 90px;
+}
+.input-medium {
+ width: 150px;
+}
+.input-large {
+ width: 210px;
+}
+.input-xlarge {
+ width: 270px;
+}
+.input-xxlarge {
+ width: 530px;
+}
+input[class*="span"],
+select[class*="span"],
+textarea[class*="span"],
+.uneditable-input[class*="span"],
+.row-fluid input[class*="span"],
+.row-fluid select[class*="span"],
+.row-fluid textarea[class*="span"],
+.row-fluid .uneditable-input[class*="span"] {
+ float: none;
+ margin-left: 0;
+}
+.input-append input[class*="span"],
+.input-append .uneditable-input[class*="span"],
+.input-prepend input[class*="span"],
+.input-prepend .uneditable-input[class*="span"],
+.row-fluid input[class*="span"],
+.row-fluid select[class*="span"],
+.row-fluid textarea[class*="span"],
+.row-fluid .uneditable-input[class*="span"],
+.row-fluid .input-prepend [class*="span"],
+.row-fluid .input-append [class*="span"] {
+ display: inline-block;
+}
+input,
+textarea,
+.uneditable-input {
+ margin-left: 0;
+}
+.controls-row [class*="span"] + [class*="span"] {
+ margin-left: 20px;
+}
+input.span12,
+textarea.span12,
+.uneditable-input.span12 {
+ width: 926px;
+}
+input.span11,
+textarea.span11,
+.uneditable-input.span11 {
+ width: 846px;
+}
+input.span10,
+textarea.span10,
+.uneditable-input.span10 {
+ width: 766px;
+}
+input.span9,
+textarea.span9,
+.uneditable-input.span9 {
+ width: 686px;
+}
+input.span8,
+textarea.span8,
+.uneditable-input.span8 {
+ width: 606px;
+}
+input.span7,
+textarea.span7,
+.uneditable-input.span7 {
+ width: 526px;
+}
+input.span6,
+textarea.span6,
+.uneditable-input.span6 {
+ width: 446px;
+}
+input.span5,
+textarea.span5,
+.uneditable-input.span5 {
+ width: 366px;
+}
+input.span4,
+textarea.span4,
+.uneditable-input.span4 {
+ width: 286px;
+}
+input.span3,
+textarea.span3,
+.uneditable-input.span3 {
+ width: 206px;
+}
+input.span2,
+textarea.span2,
+.uneditable-input.span2 {
+ width: 126px;
+}
+input.span1,
+textarea.span1,
+.uneditable-input.span1 {
+ width: 46px;
+}
+.controls-row {
+ *zoom: 1;
+}
+.controls-row:before,
+.controls-row:after {
+ display: table;
+ content: "";
+ line-height: 0;
+}
+.controls-row:after {
+ clear: both;
+}
+.controls-row [class*="span"],
+.row-fluid .controls-row [class*="span"] {
+ float: left;
+}
+.controls-row .checkbox[class*="span"],
+.controls-row .radio[class*="span"] {
+ padding-top: 5px;
+}
+input[disabled],
+select[disabled],
+textarea[disabled],
+input[readonly],
+select[readonly],
+textarea[readonly] {
+ cursor: not-allowed;
+ background-color: #eeeeee;
+}
+input[type="radio"][disabled],
+input[type="checkbox"][disabled],
+input[type="radio"][readonly],
+input[type="checkbox"][readonly] {
+ background-color: transparent;
+}
+.control-group.warning .control-label,
+.control-group.warning .help-block,
+.control-group.warning .help-inline {
+ color: #c09853;
+}
+.control-group.warning .checkbox,
+.control-group.warning .radio,
+.control-group.warning input,
+.control-group.warning select,
+.control-group.warning textarea {
+ color: #c09853;
+}
+.control-group.warning input,
+.control-group.warning select,
+.control-group.warning textarea {
+ border-color: #c09853;
+ -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
+ -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
+ box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
+}
+.control-group.warning input:focus,
+.control-group.warning select:focus,
+.control-group.warning textarea:focus {
+ border-color: #a47e3c;
+ -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #dbc59e;
+ -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #dbc59e;
+ box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #dbc59e;
+}
+.control-group.warning .input-prepend .add-on,
+.control-group.warning .input-append .add-on {
+ color: #c09853;
+ background-color: #fcf8e3;
+ border-color: #c09853;
+}
+.control-group.error .control-label,
+.control-group.error .help-block,
+.control-group.error .help-inline {
+ color: #b94a48;
+}
+.control-group.error .checkbox,
+.control-group.error .radio,
+.control-group.error input,
+.control-group.error select,
+.control-group.error textarea {
+ color: #b94a48;
+}
+.control-group.error input,
+.control-group.error select,
+.control-group.error textarea {
+ border-color: #b94a48;
+ -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
+ -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
+ box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
+}
+.control-group.error input:focus,
+.control-group.error select:focus,
+.control-group.error textarea:focus {
+ border-color: #953b39;
+ -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #d59392;
+ -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #d59392;
+ box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #d59392;
+}
+.control-group.error .input-prepend .add-on,
+.control-group.error .input-append .add-on {
+ color: #b94a48;
+ background-color: #f2dede;
+ border-color: #b94a48;
+}
+.control-group.success .control-label,
+.control-group.success .help-block,
+.control-group.success .help-inline {
+ color: #468847;
+}
+.control-group.success .checkbox,
+.control-group.success .radio,
+.control-group.success input,
+.control-group.success select,
+.control-group.success textarea {
+ color: #468847;
+}
+.control-group.success input,
+.control-group.success select,
+.control-group.success textarea {
+ border-color: #468847;
+ -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
+ -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
+ box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
+}
+.control-group.success input:focus,
+.control-group.success select:focus,
+.control-group.success textarea:focus {
+ border-color: #356635;
+ -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #7aba7b;
+ -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #7aba7b;
+ box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #7aba7b;
+}
+.control-group.success .input-prepend .add-on,
+.control-group.success .input-append .add-on {
+ color: #468847;
+ background-color: #dff0d8;
+ border-color: #468847;
+}
+.control-group.info .control-label,
+.control-group.info .help-block,
+.control-group.info .help-inline {
+ color: #3a87ad;
+}
+.control-group.info .checkbox,
+.control-group.info .radio,
+.control-group.info input,
+.control-group.info select,
+.control-group.info textarea {
+ color: #3a87ad;
+}
+.control-group.info input,
+.control-group.info select,
+.control-group.info textarea {
+ border-color: #3a87ad;
+ -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
+ -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
+ box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
+}
+.control-group.info input:focus,
+.control-group.info select:focus,
+.control-group.info textarea:focus {
+ border-color: #2d6987;
+ -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #7ab5d3;
+ -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #7ab5d3;
+ box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #7ab5d3;
+}
+.control-group.info .input-prepend .add-on,
+.control-group.info .input-append .add-on {
+ color: #3a87ad;
+ background-color: #d9edf7;
+ border-color: #3a87ad;
+}
+input:focus:invalid,
+textarea:focus:invalid,
+select:focus:invalid {
+ color: #b94a48;
+ border-color: #ee5f5b;
+}
+input:focus:invalid:focus,
+textarea:focus:invalid:focus,
+select:focus:invalid:focus {
+ border-color: #e9322d;
+ -webkit-box-shadow: 0 0 6px #f8b9b7;
+ -moz-box-shadow: 0 0 6px #f8b9b7;
+ box-shadow: 0 0 6px #f8b9b7;
+}
+.form-actions {
+ padding: 19px 20px 20px;
+ margin-top: 20px;
+ margin-bottom: 20px;
+ background-color: #f5f5f5;
+ border-top: 1px solid #e5e5e5;
+ *zoom: 1;
+}
+.form-actions:before,
+.form-actions:after {
+ display: table;
+ content: "";
+ line-height: 0;
+}
+.form-actions:after {
+ clear: both;
+}
+.help-block,
+.help-inline {
+ color: #595959;
+}
+.help-block {
+ display: block;
+ margin-bottom: 10px;
+}
+.help-inline {
+ display: inline-block;
+ *display: inline;
+ /* IE7 inline-block hack */
+
+ *zoom: 1;
+ vertical-align: middle;
+ padding-left: 5px;
+}
+.input-append,
+.input-prepend {
+ display: inline-block;
+ margin-bottom: 10px;
+ vertical-align: middle;
+ font-size: 0;
+ white-space: nowrap;
+}
+.input-append input,
+.input-prepend input,
+.input-append select,
+.input-prepend select,
+.input-append .uneditable-input,
+.input-prepend .uneditable-input,
+.input-append .dropdown-menu,
+.input-prepend .dropdown-menu,
+.input-append .popover,
+.input-prepend .popover {
+ font-size: 13px;
+}
+.input-append input,
+.input-prepend input,
+.input-append select,
+.input-prepend select,
+.input-append .uneditable-input,
+.input-prepend .uneditable-input {
+ position: relative;
+ margin-bottom: 0;
+ *margin-left: 0;
+ vertical-align: top;
+ -webkit-border-radius: 0 4px 4px 0;
+ -moz-border-radius: 0 4px 4px 0;
+ border-radius: 0 4px 4px 0;
+}
+.input-append input:focus,
+.input-prepend input:focus,
+.input-append select:focus,
+.input-prepend select:focus,
+.input-append .uneditable-input:focus,
+.input-prepend .uneditable-input:focus {
+ z-index: 2;
+}
+.input-append .add-on,
+.input-prepend .add-on {
+ display: inline-block;
+ width: auto;
+ height: 20px;
+ min-width: 16px;
+ padding: 4px 5px;
+ font-size: 13px;
+ font-weight: normal;
+ line-height: 20px;
+ text-align: center;
+ text-shadow: 0 1px 0 #ffffff;
+ background-color: #eeeeee;
+ border: 1px solid #ccc;
+}
+.input-append .add-on,
+.input-prepend .add-on,
+.input-append .btn,
+.input-prepend .btn,
+.input-append .btn-group > .dropdown-toggle,
+.input-prepend .btn-group > .dropdown-toggle {
+ vertical-align: top;
+ -webkit-border-radius: 0;
+ -moz-border-radius: 0;
+ border-radius: 0;
+}
+.input-append .active,
+.input-prepend .active {
+ background-color: #a9dba9;
+ border-color: #46a546;
+}
+.input-prepend .add-on,
+.input-prepend .btn {
+ margin-right: -1px;
+}
+.input-prepend .add-on:first-child,
+.input-prepend .btn:first-child {
+ -webkit-border-radius: 4px 0 0 4px;
+ -moz-border-radius: 4px 0 0 4px;
+ border-radius: 4px 0 0 4px;
+}
+.input-append input,
+.input-append select,
+.input-append .uneditable-input {
+ -webkit-border-radius: 4px 0 0 4px;
+ -moz-border-radius: 4px 0 0 4px;
+ border-radius: 4px 0 0 4px;
+}
+.input-append input + .btn-group .btn:last-child,
+.input-append select + .btn-group .btn:last-child,
+.input-append .uneditable-input + .btn-group .btn:last-child {
+ -webkit-border-radius: 0 4px 4px 0;
+ -moz-border-radius: 0 4px 4px 0;
+ border-radius: 0 4px 4px 0;
+}
+.input-append .add-on,
+.input-append .btn,
+.input-append .btn-group {
+ margin-left: -1px;
+}
+.input-append .add-on:last-child,
+.input-append .btn:last-child,
+.input-append .btn-group:last-child > .dropdown-toggle {
+ -webkit-border-radius: 0 4px 4px 0;
+ -moz-border-radius: 0 4px 4px 0;
+ border-radius: 0 4px 4px 0;
+}
+.input-prepend.input-append input,
+.input-prepend.input-append select,
+.input-prepend.input-append .uneditable-input {
+ -webkit-border-radius: 0;
+ -moz-border-radius: 0;
+ border-radius: 0;
+}
+.input-prepend.input-append input + .btn-group .btn,
+.input-prepend.input-append select + .btn-group .btn,
+.input-prepend.input-append .uneditable-input + .btn-group .btn {
+ -webkit-border-radius: 0 4px 4px 0;
+ -moz-border-radius: 0 4px 4px 0;
+ border-radius: 0 4px 4px 0;
+}
+.input-prepend.input-append .add-on:first-child,
+.input-prepend.input-append .btn:first-child {
+ margin-right: -1px;
+ -webkit-border-radius: 4px 0 0 4px;
+ -moz-border-radius: 4px 0 0 4px;
+ border-radius: 4px 0 0 4px;
+}
+.input-prepend.input-append .add-on:last-child,
+.input-prepend.input-append .btn:last-child {
+ margin-left: -1px;
+ -webkit-border-radius: 0 4px 4px 0;
+ -moz-border-radius: 0 4px 4px 0;
+ border-radius: 0 4px 4px 0;
+}
+.input-prepend.input-append .btn-group:first-child {
+ margin-left: 0;
+}
+input.search-query {
+ padding-right: 14px;
+ padding-right: 4px \9;
+ padding-left: 14px;
+ padding-left: 4px \9;
+ /* IE7-8 doesn't have border-radius, so don't indent the padding */
+
+ margin-bottom: 0;
+ -webkit-border-radius: 15px;
+ -moz-border-radius: 15px;
+ border-radius: 15px;
+}
+/* Allow for input prepend/append in search forms */
+.form-search .input-append .search-query,
+.form-search .input-prepend .search-query {
+ -webkit-border-radius: 0;
+ -moz-border-radius: 0;
+ border-radius: 0;
+}
+.form-search .input-append .search-query {
+ -webkit-border-radius: 14px 0 0 14px;
+ -moz-border-radius: 14px 0 0 14px;
+ border-radius: 14px 0 0 14px;
+}
+.form-search .input-append .btn {
+ -webkit-border-radius: 0 14px 14px 0;
+ -moz-border-radius: 0 14px 14px 0;
+ border-radius: 0 14px 14px 0;
+}
+.form-search .input-prepend .search-query {
+ -webkit-border-radius: 0 14px 14px 0;
+ -moz-border-radius: 0 14px 14px 0;
+ border-radius: 0 14px 14px 0;
+}
+.form-search .input-prepend .btn {
+ -webkit-border-radius: 14px 0 0 14px;
+ -moz-border-radius: 14px 0 0 14px;
+ border-radius: 14px 0 0 14px;
+}
+.form-search input,
+.form-inline input,
+.form-horizontal input,
+.form-search textarea,
+.form-inline textarea,
+.form-horizontal textarea,
+.form-search select,
+.form-inline select,
+.form-horizontal select,
+.form-search .help-inline,
+.form-inline .help-inline,
+.form-horizontal .help-inline,
+.form-search .uneditable-input,
+.form-inline .uneditable-input,
+.form-horizontal .uneditable-input,
+.form-search .input-prepend,
+.form-inline .input-prepend,
+.form-horizontal .input-prepend,
+.form-search .input-append,
+.form-inline .input-append,
+.form-horizontal .input-append {
+ display: inline-block;
+ *display: inline;
+ /* IE7 inline-block hack */
+
+ *zoom: 1;
+ margin-bottom: 0;
+ vertical-align: middle;
+}
+.form-search .hide,
+.form-inline .hide,
+.form-horizontal .hide {
+ display: none;
+}
+.form-search label,
+.form-inline label,
+.form-search .btn-group,
+.form-inline .btn-group {
+ display: inline-block;
+}
+.form-search .input-append,
+.form-inline .input-append,
+.form-search .input-prepend,
+.form-inline .input-prepend {
+ margin-bottom: 0;
+}
+.form-search .radio,
+.form-search .checkbox,
+.form-inline .radio,
+.form-inline .checkbox {
+ padding-left: 0;
+ margin-bottom: 0;
+ vertical-align: middle;
+}
+.form-search .radio input[type="radio"],
+.form-search .checkbox input[type="checkbox"],
+.form-inline .radio input[type="radio"],
+.form-inline .checkbox input[type="checkbox"] {
+ float: left;
+ margin-right: 3px;
+ margin-left: 0;
+}
+.control-group {
+ margin-bottom: 10px;
+}
+legend + .control-group {
+ margin-top: 20px;
+ -webkit-margin-top-collapse: separate;
+}
+.form-horizontal .control-group {
+ margin-bottom: 20px;
+ *zoom: 1;
+}
+.form-horizontal .control-group:before,
+.form-horizontal .control-group:after {
+ display: table;
+ content: "";
+ line-height: 0;
+}
+.form-horizontal .control-group:after {
+ clear: both;
+}
+.form-horizontal .control-label {
+ float: left;
+ width: 160px;
+ padding-top: 5px;
+ text-align: right;
+}
+.form-horizontal .controls {
+ *display: inline-block;
+ *padding-left: 20px;
+ margin-left: 180px;
+ *margin-left: 0;
+}
+.form-horizontal .controls:first-child {
+ *padding-left: 180px;
+}
+.form-horizontal .help-block {
+ margin-bottom: 0;
+}
+.form-horizontal input + .help-block,
+.form-horizontal select + .help-block,
+.form-horizontal textarea + .help-block,
+.form-horizontal .uneditable-input + .help-block,
+.form-horizontal .input-prepend + .help-block,
+.form-horizontal .input-append + .help-block {
+ margin-top: 10px;
+}
+.form-horizontal .form-actions {
+ padding-left: 180px;
+}
+table {
+ max-width: 100%;
+ background-color: transparent;
+ border-collapse: collapse;
+ border-spacing: 0;
+}
+.table {
+ width: 100%;
+ margin-bottom: 20px;
+}
+.table th,
+.table td {
+ padding: 8px;
+ line-height: 20px;
+ text-align: left;
+ vertical-align: top;
+ border-top: 1px solid #dddddd;
+}
+.table th {
+ font-weight: bold;
+}
+.table thead th {
+ vertical-align: bottom;
+}
+.table caption + thead tr:first-child th,
+.table caption + thead tr:first-child td,
+.table colgroup + thead tr:first-child th,
+.table colgroup + thead tr:first-child td,
+.table thead:first-child tr:first-child th,
+.table thead:first-child tr:first-child td {
+ border-top: 0;
+}
+.table tbody + tbody {
+ border-top: 2px solid #dddddd;
+}
+.table .table {
+ background-color: #ffffff;
+}
+.table-condensed th,
+.table-condensed td {
+ padding: 4px 5px;
+}
+.table-bordered {
+ border: 1px solid #dddddd;
+ border-collapse: separate;
+ *border-collapse: collapse;
+ border-left: 0;
+ -webkit-border-radius: 4px;
+ -moz-border-radius: 4px;
+ border-radius: 4px;
+}
+.table-bordered th,
+.table-bordered td {
+ border-left: 1px solid #dddddd;
+}
+.table-bordered caption + thead tr:first-child th,
+.table-bordered caption + tbody tr:first-child th,
+.table-bordered caption + tbody tr:first-child td,
+.table-bordered colgroup + thead tr:first-child th,
+.table-bordered colgroup + tbody tr:first-child th,
+.table-bordered colgroup + tbody tr:first-child td,
+.table-bordered thead:first-child tr:first-child th,
+.table-bordered tbody:first-child tr:first-child th,
+.table-bordered tbody:first-child tr:first-child td {
+ border-top: 0;
+}
+.table-bordered thead:first-child tr:first-child > th:first-child,
+.table-bordered tbody:first-child tr:first-child > td:first-child,
+.table-bordered tbody:first-child tr:first-child > th:first-child {
+ -webkit-border-top-left-radius: 4px;
+ -moz-border-radius-topleft: 4px;
+ border-top-left-radius: 4px;
+}
+.table-bordered thead:first-child tr:first-child > th:last-child,
+.table-bordered tbody:first-child tr:first-child > td:last-child,
+.table-bordered tbody:first-child tr:first-child > th:last-child {
+ -webkit-border-top-right-radius: 4px;
+ -moz-border-radius-topright: 4px;
+ border-top-right-radius: 4px;
+}
+.table-bordered thead:last-child tr:last-child > th:first-child,
+.table-bordered tbody:last-child tr:last-child > td:first-child,
+.table-bordered tbody:last-child tr:last-child > th:first-child,
+.table-bordered tfoot:last-child tr:last-child > td:first-child,
+.table-bordered tfoot:last-child tr:last-child > th:first-child {
+ -webkit-border-bottom-left-radius: 4px;
+ -moz-border-radius-bottomleft: 4px;
+ border-bottom-left-radius: 4px;
+}
+.table-bordered thead:last-child tr:last-child > th:last-child,
+.table-bordered tbody:last-child tr:last-child > td:last-child,
+.table-bordered tbody:last-child tr:last-child > th:last-child,
+.table-bordered tfoot:last-child tr:last-child > td:last-child,
+.table-bordered tfoot:last-child tr:last-child > th:last-child {
+ -webkit-border-bottom-right-radius: 4px;
+ -moz-border-radius-bottomright: 4px;
+ border-bottom-right-radius: 4px;
+}
+.table-bordered tfoot + tbody:last-child tr:last-child td:first-child {
+ -webkit-border-bottom-left-radius: 0;
+ -moz-border-radius-bottomleft: 0;
+ border-bottom-left-radius: 0;
+}
+.table-bordered tfoot + tbody:last-child tr:last-child td:last-child {
+ -webkit-border-bottom-right-radius: 0;
+ -moz-border-radius-bottomright: 0;
+ border-bottom-right-radius: 0;
+}
+.table-bordered caption + thead tr:first-child th:first-child,
+.table-bordered caption + tbody tr:first-child td:first-child,
+.table-bordered colgroup + thead tr:first-child th:first-child,
+.table-bordered colgroup + tbody tr:first-child td:first-child {
+ -webkit-border-top-left-radius: 4px;
+ -moz-border-radius-topleft: 4px;
+ border-top-left-radius: 4px;
+}
+.table-bordered caption + thead tr:first-child th:last-child,
+.table-bordered caption + tbody tr:first-child td:last-child,
+.table-bordered colgroup + thead tr:first-child th:last-child,
+.table-bordered colgroup + tbody tr:first-child td:last-child {
+ -webkit-border-top-right-radius: 4px;
+ -moz-border-radius-topright: 4px;
+ border-top-right-radius: 4px;
+}
+.table-striped tbody > tr:nth-child(odd) > td,
+.table-striped tbody > tr:nth-child(odd) > th {
+ background-color: #f9f9f9;
+}
+.table-hover tbody tr:hover > td,
+.table-hover tbody tr:hover > th {
+ background-color: #f5f5f5;
+}
+table td[class*="span"],
+table th[class*="span"],
+.row-fluid table td[class*="span"],
+.row-fluid table th[class*="span"] {
+ display: table-cell;
+ float: none;
+ margin-left: 0;
+}
+.table td.span1,
+.table th.span1 {
+ float: none;
+ width: 44px;
+ margin-left: 0;
+}
+.table td.span2,
+.table th.span2 {
+ float: none;
+ width: 124px;
+ margin-left: 0;
+}
+.table td.span3,
+.table th.span3 {
+ float: none;
+ width: 204px;
+ margin-left: 0;
+}
+.table td.span4,
+.table th.span4 {
+ float: none;
+ width: 284px;
+ margin-left: 0;
+}
+.table td.span5,
+.table th.span5 {
+ float: none;
+ width: 364px;
+ margin-left: 0;
+}
+.table td.span6,
+.table th.span6 {
+ float: none;
+ width: 444px;
+ margin-left: 0;
+}
+.table td.span7,
+.table th.span7 {
+ float: none;
+ width: 524px;
+ margin-left: 0;
+}
+.table td.span8,
+.table th.span8 {
+ float: none;
+ width: 604px;
+ margin-left: 0;
+}
+.table td.span9,
+.table th.span9 {
+ float: none;
+ width: 684px;
+ margin-left: 0;
+}
+.table td.span10,
+.table th.span10 {
+ float: none;
+ width: 764px;
+ margin-left: 0;
+}
+.table td.span11,
+.table th.span11 {
+ float: none;
+ width: 844px;
+ margin-left: 0;
+}
+.table td.span12,
+.table th.span12 {
+ float: none;
+ width: 924px;
+ margin-left: 0;
+}
+.table tbody tr.success > td {
+ background-color: #dff0d8;
+}
+.table tbody tr.error > td {
+ background-color: #f2dede;
+}
+.table tbody tr.warning > td {
+ background-color: #fcf8e3;
+}
+.table tbody tr.info > td {
+ background-color: #d9edf7;
+}
+.table-hover tbody tr.success:hover > td {
+ background-color: #d0e9c6;
+}
+.table-hover tbody tr.error:hover > td {
+ background-color: #ebcccc;
+}
+.table-hover tbody tr.warning:hover > td {
+ background-color: #faf2cc;
+}
+.table-hover tbody tr.info:hover > td {
+ background-color: #c4e3f3;
+}
+/*@import "bootstrap/sprites.less";*/
+.dropup,
+.dropdown {
+ position: relative;
+}
+.dropdown-toggle {
+ *margin-bottom: -3px;
+}
+.dropdown-toggle:active,
+.open .dropdown-toggle {
+ outline: 0;
+}
+.caret {
+ display: inline-block;
+ width: 0;
+ height: 0;
+ vertical-align: top;
+ border-top: 4px solid #000000;
+ border-right: 4px solid transparent;
+ border-left: 4px solid transparent;
+ content: "";
+}
+.dropdown .caret {
+ margin-top: 8px;
+ margin-left: 2px;
+}
+.dropdown-menu {
+ position: absolute;
+ top: 100%;
+ left: 0;
+ z-index: 1000;
+ display: none;
+ float: left;
+ min-width: 160px;
+ padding: 5px 0;
+ margin: 2px 0 0;
+ list-style: none;
+ background-color: #ffffff;
+ border: 1px solid #ccc;
+ border: 1px solid rgba(0, 0, 0, 0.2);
+ *border-right-width: 2px;
+ *border-bottom-width: 2px;
+ -webkit-border-radius: 6px;
+ -moz-border-radius: 6px;
+ border-radius: 6px;
+ -webkit-box-shadow: 0 5px 10px rgba(0, 0, 0, 0.2);
+ -moz-box-shadow: 0 5px 10px rgba(0, 0, 0, 0.2);
+ box-shadow: 0 5px 10px rgba(0, 0, 0, 0.2);
+ -webkit-background-clip: padding-box;
+ -moz-background-clip: padding;
+ background-clip: padding-box;
+}
+.dropdown-menu.pull-right {
+ right: 0;
+ left: auto;
+}
+.dropdown-menu .divider {
+ *width: 100%;
+ height: 1px;
+ margin: 9px 1px;
+ *margin: -5px 0 5px;
+ overflow: hidden;
+ background-color: #e5e5e5;
+ border-bottom: 1px solid #ffffff;
+}
+.dropdown-menu > li > a {
+ display: block;
+ padding: 3px 20px;
+ clear: both;
+ font-weight: normal;
+ line-height: 20px;
+ color: #333333;
+ white-space: nowrap;
+}
+.dropdown-menu > li > a:hover,
+.dropdown-menu > li > a:focus,
+.dropdown-submenu:hover > a,
+.dropdown-submenu:focus > a {
+ text-decoration: none;
+ color: #ffffff;
+ background-color: #0081c2;
+ background-image: -moz-linear-gradient(top, #0088cc, #0077b3);
+ background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#0088cc), to(#0077b3));
+ background-image: -webkit-linear-gradient(top, #0088cc, #0077b3);
+ background-image: -o-linear-gradient(top, #0088cc, #0077b3);
+ background-image: linear-gradient(to bottom, #0088cc, #0077b3);
+ background-repeat: repeat-x;
+ filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff0088cc', endColorstr='#ff0077b3', GradientType=0);
+}
+.dropdown-menu > .active > a,
+.dropdown-menu > .active > a:hover,
+.dropdown-menu > .active > a:focus {
+ color: #ffffff;
+ text-decoration: none;
+ outline: 0;
+ background-color: #0081c2;
+ background-image: -moz-linear-gradient(top, #0088cc, #0077b3);
+ background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#0088cc), to(#0077b3));
+ background-image: -webkit-linear-gradient(top, #0088cc, #0077b3);
+ background-image: -o-linear-gradient(top, #0088cc, #0077b3);
+ background-image: linear-gradient(to bottom, #0088cc, #0077b3);
+ background-repeat: repeat-x;
+ filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff0088cc', endColorstr='#ff0077b3', GradientType=0);
+}
+.dropdown-menu > .disabled > a,
+.dropdown-menu > .disabled > a:hover,
+.dropdown-menu > .disabled > a:focus {
+ color: #999999;
+}
+.dropdown-menu > .disabled > a:hover,
+.dropdown-menu > .disabled > a:focus {
+ text-decoration: none;
+ background-color: transparent;
+ background-image: none;
+ filter: progid:DXImageTransform.Microsoft.gradient(enabled = false);
+ cursor: default;
+}
+.open {
+ *z-index: 1000;
+}
+.open > .dropdown-menu {
+ display: block;
+}
+.pull-right > .dropdown-menu {
+ right: 0;
+ left: auto;
+}
+.dropup .caret,
+.navbar-fixed-bottom .dropdown .caret {
+ border-top: 0;
+ border-bottom: 4px solid #000000;
+ content: "";
+}
+.dropup .dropdown-menu,
+.navbar-fixed-bottom .dropdown .dropdown-menu {
+ top: auto;
+ bottom: 100%;
+ margin-bottom: 1px;
+}
+.dropdown-submenu {
+ position: relative;
+}
+.dropdown-submenu > .dropdown-menu {
+ top: 0;
+ left: 100%;
+ margin-top: -6px;
+ margin-left: -1px;
+ -webkit-border-radius: 0 6px 6px 6px;
+ -moz-border-radius: 0 6px 6px 6px;
+ border-radius: 0 6px 6px 6px;
+}
+.dropdown-submenu:hover > .dropdown-menu {
+ display: block;
+}
+.dropup .dropdown-submenu > .dropdown-menu {
+ top: auto;
+ bottom: 0;
+ margin-top: 0;
+ margin-bottom: -2px;
+ -webkit-border-radius: 5px 5px 5px 0;
+ -moz-border-radius: 5px 5px 5px 0;
+ border-radius: 5px 5px 5px 0;
+}
+.dropdown-submenu > a:after {
+ display: block;
+ content: " ";
+ float: right;
+ width: 0;
+ height: 0;
+ border-color: transparent;
+ border-style: solid;
+ border-width: 5px 0 5px 5px;
+ border-left-color: #cccccc;
+ margin-top: 5px;
+ margin-right: -10px;
+}
+.dropdown-submenu:hover > a:after {
+ border-left-color: #ffffff;
+}
+.dropdown-submenu.pull-left {
+ float: none;
+}
+.dropdown-submenu.pull-left > .dropdown-menu {
+ left: -100%;
+ margin-left: 10px;
+ -webkit-border-radius: 6px 0 6px 6px;
+ -moz-border-radius: 6px 0 6px 6px;
+ border-radius: 6px 0 6px 6px;
+}
+.dropdown .dropdown-menu .nav-header {
+ padding-left: 20px;
+ padding-right: 20px;
+}
+.typeahead {
+ z-index: 1051;
+ margin-top: 2px;
+ -webkit-border-radius: 4px;
+ -moz-border-radius: 4px;
+ border-radius: 4px;
+}
+.well {
+ min-height: 20px;
+ padding: 19px;
+ margin-bottom: 20px;
+ background-color: #f5f5f5;
+ border: 1px solid #e3e3e3;
+ -webkit-border-radius: 4px;
+ -moz-border-radius: 4px;
+ border-radius: 4px;
+ -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.05);
+ -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.05);
+ box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.05);
+}
+.well blockquote {
+ border-color: #ddd;
+ border-color: rgba(0, 0, 0, 0.15);
+}
+.well-large {
+ padding: 24px;
+ -webkit-border-radius: 6px;
+ -moz-border-radius: 6px;
+ border-radius: 6px;
+}
+.well-small {
+ padding: 9px;
+ -webkit-border-radius: 3px;
+ -moz-border-radius: 3px;
+ border-radius: 3px;
+}
+/*@import "bootstrap/component-animations.less";*/
+/*@import "bootstrap/close.less";*/
+.btn {
+ display: inline-block;
+ *display: inline;
+ /* IE7 inline-block hack */
+
+ *zoom: 1;
+ padding: 4px 12px;
+ margin-bottom: 0;
+ font-size: 13px;
+ line-height: 20px;
+ text-align: center;
+ vertical-align: middle;
+ cursor: pointer;
+ color: #333333;
+ text-shadow: 0 1px 1px rgba(255, 255, 255, 0.75);
+ background-color: #f5f5f5;
+ background-image: -moz-linear-gradient(top, #ffffff, #e6e6e6);
+ background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#ffffff), to(#e6e6e6));
+ background-image: -webkit-linear-gradient(top, #ffffff, #e6e6e6);
+ background-image: -o-linear-gradient(top, #ffffff, #e6e6e6);
+ background-image: linear-gradient(to bottom, #ffffff, #e6e6e6);
+ background-repeat: repeat-x;
+ filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffffffff', endColorstr='#ffe6e6e6', GradientType=0);
+ border-color: #e6e6e6 #e6e6e6 #bfbfbf;
+ border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
+ *background-color: #e6e6e6;
+ /* Darken IE7 buttons by default so they stand out more given they won't have borders */
+
+ filter: progid:DXImageTransform.Microsoft.gradient(enabled = false);
+ border: 1px solid #cccccc;
+ *border: 0;
+ border-bottom-color: #b3b3b3;
+ -webkit-border-radius: 4px;
+ -moz-border-radius: 4px;
+ border-radius: 4px;
+ *margin-left: .3em;
+ -webkit-box-shadow: inset 0 1px 0 rgba(255,255,255,.2), 0 1px 2px rgba(0,0,0,.05);
+ -moz-box-shadow: inset 0 1px 0 rgba(255,255,255,.2), 0 1px 2px rgba(0,0,0,.05);
+ box-shadow: inset 0 1px 0 rgba(255,255,255,.2), 0 1px 2px rgba(0,0,0,.05);
+}
+.btn:hover,
+.btn:focus,
+.btn:active,
+.btn.active,
+.btn.disabled,
+.btn[disabled] {
+ color: #333333;
+ background-color: #e6e6e6;
+ *background-color: #d9d9d9;
+}
+.btn:active,
+.btn.active {
+ background-color: #cccccc \9;
+}
+.btn:first-child {
+ *margin-left: 0;
+}
+.btn:hover,
+.btn:focus {
+ color: #333333;
+ text-decoration: none;
+ background-position: 0 -15px;
+ -webkit-transition: background-position 0.1s linear;
+ -moz-transition: background-position 0.1s linear;
+ -o-transition: background-position 0.1s linear;
+ transition: background-position 0.1s linear;
+}
+.btn:focus {
+ outline: thin dotted #333;
+ outline: 5px auto -webkit-focus-ring-color;
+ outline-offset: -2px;
+}
+.btn.active,
+.btn:active {
+ background-image: none;
+ outline: 0;
+ -webkit-box-shadow: inset 0 2px 4px rgba(0,0,0,.15), 0 1px 2px rgba(0,0,0,.05);
+ -moz-box-shadow: inset 0 2px 4px rgba(0,0,0,.15), 0 1px 2px rgba(0,0,0,.05);
+ box-shadow: inset 0 2px 4px rgba(0,0,0,.15), 0 1px 2px rgba(0,0,0,.05);
+}
+.btn.disabled,
+.btn[disabled] {
+ cursor: default;
+ background-image: none;
+ opacity: 0.65;
+ filter: alpha(opacity=65);
+ -webkit-box-shadow: none;
+ -moz-box-shadow: none;
+ box-shadow: none;
+}
+.btn-large {
+ padding: 11px 19px;
+ font-size: 16.25px;
+ -webkit-border-radius: 6px;
+ -moz-border-radius: 6px;
+ border-radius: 6px;
+}
+.btn-large [class^="icon-"],
+.btn-large [class*=" icon-"] {
+ margin-top: 4px;
+}
+.btn-small {
+ padding: 2px 10px;
+ font-size: 11.049999999999999px;
+ -webkit-border-radius: 3px;
+ -moz-border-radius: 3px;
+ border-radius: 3px;
+}
+.btn-small [class^="icon-"],
+.btn-small [class*=" icon-"] {
+ margin-top: 0;
+}
+.btn-mini [class^="icon-"],
+.btn-mini [class*=" icon-"] {
+ margin-top: -1px;
+}
+.btn-mini {
+ padding: 0 6px;
+ font-size: 9.75px;
+ -webkit-border-radius: 3px;
+ -moz-border-radius: 3px;
+ border-radius: 3px;
+}
+.btn-block {
+ display: block;
+ width: 100%;
+ padding-left: 0;
+ padding-right: 0;
+ -webkit-box-sizing: border-box;
+ -moz-box-sizing: border-box;
+ box-sizing: border-box;
+}
+.btn-block + .btn-block {
+ margin-top: 5px;
+}
+input[type="submit"].btn-block,
+input[type="reset"].btn-block,
+input[type="button"].btn-block {
+ width: 100%;
+}
+.btn-primary.active,
+.btn-warning.active,
+.btn-danger.active,
+.btn-success.active,
+.btn-info.active,
+.btn-inverse.active {
+ color: rgba(255, 255, 255, 0.75);
+}
+.btn-primary {
+ color: #ffffff;
+ text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25);
+ background-color: #006dcc;
+ background-image: -moz-linear-gradient(top, #0088cc, #0044cc);
+ background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#0088cc), to(#0044cc));
+ background-image: -webkit-linear-gradient(top, #0088cc, #0044cc);
+ background-image: -o-linear-gradient(top, #0088cc, #0044cc);
+ background-image: linear-gradient(to bottom, #0088cc, #0044cc);
+ background-repeat: repeat-x;
+ filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff0088cc', endColorstr='#ff0044cc', GradientType=0);
+ border-color: #0044cc #0044cc #002a80;
+ border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
+ *background-color: #0044cc;
+ /* Darken IE7 buttons by default so they stand out more given they won't have borders */
+
+ filter: progid:DXImageTransform.Microsoft.gradient(enabled = false);
+}
+.btn-primary:hover,
+.btn-primary:focus,
+.btn-primary:active,
+.btn-primary.active,
+.btn-primary.disabled,
+.btn-primary[disabled] {
+ color: #ffffff;
+ background-color: #0044cc;
+ *background-color: #003bb3;
+}
+.btn-primary:active,
+.btn-primary.active {
+ background-color: #003399 \9;
+}
+.btn-warning {
+ color: #ffffff;
+ text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25);
+ background-color: #faa732;
+ background-image: -moz-linear-gradient(top, #fbb450, #f89406);
+ background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#fbb450), to(#f89406));
+ background-image: -webkit-linear-gradient(top, #fbb450, #f89406);
+ background-image: -o-linear-gradient(top, #fbb450, #f89406);
+ background-image: linear-gradient(to bottom, #fbb450, #f89406);
+ background-repeat: repeat-x;
+ filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#fffbb450', endColorstr='#fff89406', GradientType=0);
+ border-color: #f89406 #f89406 #ad6704;
+ border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
+ *background-color: #f89406;
+ /* Darken IE7 buttons by default so they stand out more given they won't have borders */
+
+ filter: progid:DXImageTransform.Microsoft.gradient(enabled = false);
+}
+.btn-warning:hover,
+.btn-warning:focus,
+.btn-warning:active,
+.btn-warning.active,
+.btn-warning.disabled,
+.btn-warning[disabled] {
+ color: #ffffff;
+ background-color: #f89406;
+ *background-color: #df8505;
+}
+.btn-warning:active,
+.btn-warning.active {
+ background-color: #c67605 \9;
+}
+.btn-danger {
+ color: #ffffff;
+ text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25);
+ background-color: #da4f49;
+ background-image: -moz-linear-gradient(top, #ee5f5b, #bd362f);
+ background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#ee5f5b), to(#bd362f));
+ background-image: -webkit-linear-gradient(top, #ee5f5b, #bd362f);
+ background-image: -o-linear-gradient(top, #ee5f5b, #bd362f);
+ background-image: linear-gradient(to bottom, #ee5f5b, #bd362f);
+ background-repeat: repeat-x;
+ filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffee5f5b', endColorstr='#ffbd362f', GradientType=0);
+ border-color: #bd362f #bd362f #802420;
+ border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
+ *background-color: #bd362f;
+ /* Darken IE7 buttons by default so they stand out more given they won't have borders */
+
+ filter: progid:DXImageTransform.Microsoft.gradient(enabled = false);
+}
+.btn-danger:hover,
+.btn-danger:focus,
+.btn-danger:active,
+.btn-danger.active,
+.btn-danger.disabled,
+.btn-danger[disabled] {
+ color: #ffffff;
+ background-color: #bd362f;
+ *background-color: #a9302a;
+}
+.btn-danger:active,
+.btn-danger.active {
+ background-color: #942a25 \9;
+}
+.btn-success {
+ color: #ffffff;
+ text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25);
+ background-color: #5bb75b;
+ background-image: -moz-linear-gradient(top, #62c462, #51a351);
+ background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#62c462), to(#51a351));
+ background-image: -webkit-linear-gradient(top, #62c462, #51a351);
+ background-image: -o-linear-gradient(top, #62c462, #51a351);
+ background-image: linear-gradient(to bottom, #62c462, #51a351);
+ background-repeat: repeat-x;
+ filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff62c462', endColorstr='#ff51a351', GradientType=0);
+ border-color: #51a351 #51a351 #387038;
+ border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
+ *background-color: #51a351;
+ /* Darken IE7 buttons by default so they stand out more given they won't have borders */
+
+ filter: progid:DXImageTransform.Microsoft.gradient(enabled = false);
+}
+.btn-success:hover,
+.btn-success:focus,
+.btn-success:active,
+.btn-success.active,
+.btn-success.disabled,
+.btn-success[disabled] {
+ color: #ffffff;
+ background-color: #51a351;
+ *background-color: #499249;
+}
+.btn-success:active,
+.btn-success.active {
+ background-color: #408140 \9;
+}
+.btn-info {
+ color: #ffffff;
+ text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25);
+ background-color: #49afcd;
+ background-image: -moz-linear-gradient(top, #5bc0de, #2f96b4);
+ background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#5bc0de), to(#2f96b4));
+ background-image: -webkit-linear-gradient(top, #5bc0de, #2f96b4);
+ background-image: -o-linear-gradient(top, #5bc0de, #2f96b4);
+ background-image: linear-gradient(to bottom, #5bc0de, #2f96b4);
+ background-repeat: repeat-x;
+ filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff5bc0de', endColorstr='#ff2f96b4', GradientType=0);
+ border-color: #2f96b4 #2f96b4 #1f6377;
+ border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
+ *background-color: #2f96b4;
+ /* Darken IE7 buttons by default so they stand out more given they won't have borders */
+
+ filter: progid:DXImageTransform.Microsoft.gradient(enabled = false);
+}
+.btn-info:hover,
+.btn-info:focus,
+.btn-info:active,
+.btn-info.active,
+.btn-info.disabled,
+.btn-info[disabled] {
+ color: #ffffff;
+ background-color: #2f96b4;
+ *background-color: #2a85a0;
+}
+.btn-info:active,
+.btn-info.active {
+ background-color: #24748c \9;
+}
+.btn-inverse {
+ color: #ffffff;
+ text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25);
+ background-color: #363636;
+ background-image: -moz-linear-gradient(top, #444444, #222222);
+ background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#444444), to(#222222));
+ background-image: -webkit-linear-gradient(top, #444444, #222222);
+ background-image: -o-linear-gradient(top, #444444, #222222);
+ background-image: linear-gradient(to bottom, #444444, #222222);
+ background-repeat: repeat-x;
+ filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff444444', endColorstr='#ff222222', GradientType=0);
+ border-color: #222222 #222222 #000000;
+ border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
+ *background-color: #222222;
+ /* Darken IE7 buttons by default so they stand out more given they won't have borders */
+
+ filter: progid:DXImageTransform.Microsoft.gradient(enabled = false);
+}
+.btn-inverse:hover,
+.btn-inverse:focus,
+.btn-inverse:active,
+.btn-inverse.active,
+.btn-inverse.disabled,
+.btn-inverse[disabled] {
+ color: #ffffff;
+ background-color: #222222;
+ *background-color: #151515;
+}
+.btn-inverse:active,
+.btn-inverse.active {
+ background-color: #080808 \9;
+}
+button.btn,
+input[type="submit"].btn {
+ *padding-top: 3px;
+ *padding-bottom: 3px;
+}
+button.btn::-moz-focus-inner,
+input[type="submit"].btn::-moz-focus-inner {
+ padding: 0;
+ border: 0;
+}
+button.btn.btn-large,
+input[type="submit"].btn.btn-large {
+ *padding-top: 7px;
+ *padding-bottom: 7px;
+}
+button.btn.btn-small,
+input[type="submit"].btn.btn-small {
+ *padding-top: 3px;
+ *padding-bottom: 3px;
+}
+button.btn.btn-mini,
+input[type="submit"].btn.btn-mini {
+ *padding-top: 1px;
+ *padding-bottom: 1px;
+}
+.btn-link,
+.btn-link:active,
+.btn-link[disabled] {
+ background-color: transparent;
+ background-image: none;
+ -webkit-box-shadow: none;
+ -moz-box-shadow: none;
+ box-shadow: none;
+}
+.btn-link {
+ border-color: transparent;
+ cursor: pointer;
+ color: #0088cc;
+ -webkit-border-radius: 0;
+ -moz-border-radius: 0;
+ border-radius: 0;
+}
+.btn-link:hover,
+.btn-link:focus {
+ color: #005580;
+ text-decoration: underline;
+ background-color: transparent;
+}
+.btn-link[disabled]:hover,
+.btn-link[disabled]:focus {
+ color: #333333;
+ text-decoration: none;
+}
+.btn-group {
+ position: relative;
+ display: inline-block;
+ *display: inline;
+ /* IE7 inline-block hack */
+
+ *zoom: 1;
+ font-size: 0;
+ vertical-align: middle;
+ white-space: nowrap;
+ *margin-left: .3em;
+}
+.btn-group:first-child {
+ *margin-left: 0;
+}
+.btn-group + .btn-group {
+ margin-left: 5px;
+}
+.btn-toolbar {
+ font-size: 0;
+ margin-top: 10px;
+ margin-bottom: 10px;
+}
+.btn-toolbar > .btn + .btn,
+.btn-toolbar > .btn-group + .btn,
+.btn-toolbar > .btn + .btn-group {
+ margin-left: 5px;
+}
+.btn-group > .btn {
+ position: relative;
+ -webkit-border-radius: 0;
+ -moz-border-radius: 0;
+ border-radius: 0;
+}
+.btn-group > .btn + .btn {
+ margin-left: -1px;
+}
+.btn-group > .btn,
+.btn-group > .dropdown-menu,
+.btn-group > .popover {
+ font-size: 13px;
+}
+.btn-group > .btn-mini {
+ font-size: 9.75px;
+}
+.btn-group > .btn-small {
+ font-size: 11.049999999999999px;
+}
+.btn-group > .btn-large {
+ font-size: 16.25px;
+}
+.btn-group > .btn:first-child {
+ margin-left: 0;
+ -webkit-border-top-left-radius: 4px;
+ -moz-border-radius-topleft: 4px;
+ border-top-left-radius: 4px;
+ -webkit-border-bottom-left-radius: 4px;
+ -moz-border-radius-bottomleft: 4px;
+ border-bottom-left-radius: 4px;
+}
+.btn-group > .btn:last-child,
+.btn-group > .dropdown-toggle {
+ -webkit-border-top-right-radius: 4px;
+ -moz-border-radius-topright: 4px;
+ border-top-right-radius: 4px;
+ -webkit-border-bottom-right-radius: 4px;
+ -moz-border-radius-bottomright: 4px;
+ border-bottom-right-radius: 4px;
+}
+.btn-group > .btn.large:first-child {
+ margin-left: 0;
+ -webkit-border-top-left-radius: 6px;
+ -moz-border-radius-topleft: 6px;
+ border-top-left-radius: 6px;
+ -webkit-border-bottom-left-radius: 6px;
+ -moz-border-radius-bottomleft: 6px;
+ border-bottom-left-radius: 6px;
+}
+.btn-group > .btn.large:last-child,
+.btn-group > .large.dropdown-toggle {
+ -webkit-border-top-right-radius: 6px;
+ -moz-border-radius-topright: 6px;
+ border-top-right-radius: 6px;
+ -webkit-border-bottom-right-radius: 6px;
+ -moz-border-radius-bottomright: 6px;
+ border-bottom-right-radius: 6px;
+}
+.btn-group > .btn:hover,
+.btn-group > .btn:focus,
+.btn-group > .btn:active,
+.btn-group > .btn.active {
+ z-index: 2;
+}
+.btn-group .dropdown-toggle:active,
+.btn-group.open .dropdown-toggle {
+ outline: 0;
+}
+.btn-group > .btn + .dropdown-toggle {
+ padding-left: 8px;
+ padding-right: 8px;
+ -webkit-box-shadow: inset 1px 0 0 rgba(255,255,255,.125), inset 0 1px 0 rgba(255,255,255,.2), 0 1px 2px rgba(0,0,0,.05);
+ -moz-box-shadow: inset 1px 0 0 rgba(255,255,255,.125), inset 0 1px 0 rgba(255,255,255,.2), 0 1px 2px rgba(0,0,0,.05);
+ box-shadow: inset 1px 0 0 rgba(255,255,255,.125), inset 0 1px 0 rgba(255,255,255,.2), 0 1px 2px rgba(0,0,0,.05);
+ *padding-top: 5px;
+ *padding-bottom: 5px;
+}
+.btn-group > .btn-mini + .dropdown-toggle {
+ padding-left: 5px;
+ padding-right: 5px;
+ *padding-top: 2px;
+ *padding-bottom: 2px;
+}
+.btn-group > .btn-small + .dropdown-toggle {
+ *padding-top: 5px;
+ *padding-bottom: 4px;
+}
+.btn-group > .btn-large + .dropdown-toggle {
+ padding-left: 12px;
+ padding-right: 12px;
+ *padding-top: 7px;
+ *padding-bottom: 7px;
+}
+.btn-group.open .dropdown-toggle {
+ background-image: none;
+ -webkit-box-shadow: inset 0 2px 4px rgba(0,0,0,.15), 0 1px 2px rgba(0,0,0,.05);
+ -moz-box-shadow: inset 0 2px 4px rgba(0,0,0,.15), 0 1px 2px rgba(0,0,0,.05);
+ box-shadow: inset 0 2px 4px rgba(0,0,0,.15), 0 1px 2px rgba(0,0,0,.05);
+}
+.btn-group.open .btn.dropdown-toggle {
+ background-color: #e6e6e6;
+}
+.btn-group.open .btn-primary.dropdown-toggle {
+ background-color: #0044cc;
+}
+.btn-group.open .btn-warning.dropdown-toggle {
+ background-color: #f89406;
+}
+.btn-group.open .btn-danger.dropdown-toggle {
+ background-color: #bd362f;
+}
+.btn-group.open .btn-success.dropdown-toggle {
+ background-color: #51a351;
+}
+.btn-group.open .btn-info.dropdown-toggle {
+ background-color: #2f96b4;
+}
+.btn-group.open .btn-inverse.dropdown-toggle {
+ background-color: #222222;
+}
+.btn .caret {
+ margin-top: 8px;
+ margin-left: 0;
+}
+.btn-large .caret {
+ margin-top: 6px;
+}
+.btn-large .caret {
+ border-left-width: 5px;
+ border-right-width: 5px;
+ border-top-width: 5px;
+}
+.btn-mini .caret,
+.btn-small .caret {
+ margin-top: 8px;
+}
+.dropup .btn-large .caret {
+ border-bottom-width: 5px;
+}
+.btn-primary .caret,
+.btn-warning .caret,
+.btn-danger .caret,
+.btn-info .caret,
+.btn-success .caret,
+.btn-inverse .caret {
+ border-top-color: #ffffff;
+ border-bottom-color: #ffffff;
+}
+.btn-group-vertical {
+ display: inline-block;
+ *display: inline;
+ /* IE7 inline-block hack */
+
+ *zoom: 1;
+}
+.btn-group-vertical > .btn {
+ display: block;
+ float: none;
+ max-width: 100%;
+ -webkit-border-radius: 0;
+ -moz-border-radius: 0;
+ border-radius: 0;
+}
+.btn-group-vertical > .btn + .btn {
+ margin-left: 0;
+ margin-top: -1px;
+}
+.btn-group-vertical > .btn:first-child {
+ -webkit-border-radius: 4px 4px 0 0;
+ -moz-border-radius: 4px 4px 0 0;
+ border-radius: 4px 4px 0 0;
+}
+.btn-group-vertical > .btn:last-child {
+ -webkit-border-radius: 0 0 4px 4px;
+ -moz-border-radius: 0 0 4px 4px;
+ border-radius: 0 0 4px 4px;
+}
+.btn-group-vertical > .btn-large:first-child {
+ -webkit-border-radius: 6px 6px 0 0;
+ -moz-border-radius: 6px 6px 0 0;
+ border-radius: 6px 6px 0 0;
+}
+.btn-group-vertical > .btn-large:last-child {
+ -webkit-border-radius: 0 0 6px 6px;
+ -moz-border-radius: 0 0 6px 6px;
+ border-radius: 0 0 6px 6px;
+}
+.alert {
+ padding: 8px 35px 8px 14px;
+ margin-bottom: 20px;
+ text-shadow: 0 1px 0 rgba(255, 255, 255, 0.5);
+ background-color: #fcf8e3;
+ border: 1px solid #fbeed5;
+ -webkit-border-radius: 4px;
+ -moz-border-radius: 4px;
+ border-radius: 4px;
+}
+.alert,
+.alert h4 {
+ color: #c09853;
+}
+.alert h4 {
+ margin: 0;
+}
+.alert .close {
+ position: relative;
+ top: -2px;
+ right: -21px;
+ line-height: 20px;
+}
+.alert-success {
+ background-color: #dff0d8;
+ border-color: #d6e9c6;
+ color: #468847;
+}
+.alert-success h4 {
+ color: #468847;
+}
+.alert-danger,
+.alert-error {
+ background-color: #f2dede;
+ border-color: #eed3d7;
+ color: #b94a48;
+}
+.alert-danger h4,
+.alert-error h4 {
+ color: #b94a48;
+}
+.alert-info {
+ background-color: #d9edf7;
+ border-color: #bce8f1;
+ color: #3a87ad;
+}
+.alert-info h4 {
+ color: #3a87ad;
+}
+.alert-block {
+ padding-top: 14px;
+ padding-bottom: 14px;
+}
+.alert-block > p,
+.alert-block > ul {
+ margin-bottom: 0;
+}
+.alert-block p + p {
+ margin-top: 5px;
+}
+.nav {
+ margin-left: 0;
+ margin-bottom: 20px;
+ list-style: none;
+}
+.nav > li > a {
+ display: block;
+}
+.nav > li > a:hover,
+.nav > li > a:focus {
+ text-decoration: none;
+ background-color: #eeeeee;
+}
+.nav > li > a > img {
+ max-width: none;
+}
+.nav > .pull-right {
+ float: right;
+}
+.nav-header {
+ display: block;
+ padding: 3px 15px;
+ font-size: 11px;
+ font-weight: bold;
+ line-height: 20px;
+ color: #999999;
+ text-shadow: 0 1px 0 rgba(255, 255, 255, 0.5);
+ text-transform: uppercase;
+}
+.nav li + .nav-header {
+ margin-top: 9px;
+}
+.nav-list {
+ padding-left: 15px;
+ padding-right: 15px;
+ margin-bottom: 0;
+}
+.nav-list > li > a,
+.nav-list .nav-header {
+ margin-left: -15px;
+ margin-right: -15px;
+ text-shadow: 0 1px 0 rgba(255, 255, 255, 0.5);
+}
+.nav-list > li > a {
+ padding: 3px 15px;
+}
+.nav-list > .active > a,
+.nav-list > .active > a:hover,
+.nav-list > .active > a:focus {
+ color: #ffffff;
+ text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.2);
+ background-color: #0088cc;
+}
+.nav-list [class^="icon-"],
+.nav-list [class*=" icon-"] {
+ margin-right: 2px;
+}
+.nav-list .divider {
+ *width: 100%;
+ height: 1px;
+ margin: 9px 1px;
+ *margin: -5px 0 5px;
+ overflow: hidden;
+ background-color: #e5e5e5;
+ border-bottom: 1px solid #ffffff;
+}
+.nav-tabs,
+.nav-pills {
+ *zoom: 1;
+}
+.nav-tabs:before,
+.nav-pills:before,
+.nav-tabs:after,
+.nav-pills:after {
+ display: table;
+ content: "";
+ line-height: 0;
+}
+.nav-tabs:after,
+.nav-pills:after {
+ clear: both;
+}
+.nav-tabs > li,
+.nav-pills > li {
+ float: left;
+}
+.nav-tabs > li > a,
+.nav-pills > li > a {
+ padding-right: 12px;
+ padding-left: 12px;
+ margin-right: 2px;
+ line-height: 14px;
+}
+.nav-tabs {
+ border-bottom: 1px solid #ddd;
+}
+.nav-tabs > li {
+ margin-bottom: -1px;
+}
+.nav-tabs > li > a {
+ padding-top: 8px;
+ padding-bottom: 8px;
+ line-height: 20px;
+ border: 1px solid transparent;
+ -webkit-border-radius: 4px 4px 0 0;
+ -moz-border-radius: 4px 4px 0 0;
+ border-radius: 4px 4px 0 0;
+}
+.nav-tabs > li > a:hover,
+.nav-tabs > li > a:focus {
+ border-color: #eeeeee #eeeeee #dddddd;
+}
+.nav-tabs > .active > a,
+.nav-tabs > .active > a:hover,
+.nav-tabs > .active > a:focus {
+ color: #555555;
+ background-color: #ffffff;
+ border: 1px solid #ddd;
+ border-bottom-color: transparent;
+ cursor: default;
+}
+.nav-pills > li > a {
+ padding-top: 8px;
+ padding-bottom: 8px;
+ margin-top: 2px;
+ margin-bottom: 2px;
+ -webkit-border-radius: 5px;
+ -moz-border-radius: 5px;
+ border-radius: 5px;
+}
+.nav-pills > .active > a,
+.nav-pills > .active > a:hover,
+.nav-pills > .active > a:focus {
+ color: #ffffff;
+ background-color: #0088cc;
+}
+.nav-stacked > li {
+ float: none;
+}
+.nav-stacked > li > a {
+ margin-right: 0;
+}
+.nav-tabs.nav-stacked {
+ border-bottom: 0;
+}
+.nav-tabs.nav-stacked > li > a {
+ border: 1px solid #ddd;
+ -webkit-border-radius: 0;
+ -moz-border-radius: 0;
+ border-radius: 0;
+}
+.nav-tabs.nav-stacked > li:first-child > a {
+ -webkit-border-top-right-radius: 4px;
+ -moz-border-radius-topright: 4px;
+ border-top-right-radius: 4px;
+ -webkit-border-top-left-radius: 4px;
+ -moz-border-radius-topleft: 4px;
+ border-top-left-radius: 4px;
+}
+.nav-tabs.nav-stacked > li:last-child > a {
+ -webkit-border-bottom-right-radius: 4px;
+ -moz-border-radius-bottomright: 4px;
+ border-bottom-right-radius: 4px;
+ -webkit-border-bottom-left-radius: 4px;
+ -moz-border-radius-bottomleft: 4px;
+ border-bottom-left-radius: 4px;
+}
+.nav-tabs.nav-stacked > li > a:hover,
+.nav-tabs.nav-stacked > li > a:focus {
+ border-color: #ddd;
+ z-index: 2;
+}
+.nav-pills.nav-stacked > li > a {
+ margin-bottom: 3px;
+}
+.nav-pills.nav-stacked > li:last-child > a {
+ margin-bottom: 1px;
+}
+.nav-tabs .dropdown-menu {
+ -webkit-border-radius: 0 0 6px 6px;
+ -moz-border-radius: 0 0 6px 6px;
+ border-radius: 0 0 6px 6px;
+}
+.nav-pills .dropdown-menu {
+ -webkit-border-radius: 6px;
+ -moz-border-radius: 6px;
+ border-radius: 6px;
+}
+.nav .dropdown-toggle .caret {
+ border-top-color: #0088cc;
+ border-bottom-color: #0088cc;
+ margin-top: 6px;
+}
+.nav .dropdown-toggle:hover .caret,
+.nav .dropdown-toggle:focus .caret {
+ border-top-color: #005580;
+ border-bottom-color: #005580;
+}
+/* move down carets for tabs */
+.nav-tabs .dropdown-toggle .caret {
+ margin-top: 8px;
+}
+.nav .active .dropdown-toggle .caret {
+ border-top-color: #fff;
+ border-bottom-color: #fff;
+}
+.nav-tabs .active .dropdown-toggle .caret {
+ border-top-color: #555555;
+ border-bottom-color: #555555;
+}
+.nav > .dropdown.active > a:hover,
+.nav > .dropdown.active > a:focus {
+ cursor: pointer;
+}
+.nav-tabs .open .dropdown-toggle,
+.nav-pills .open .dropdown-toggle,
+.nav > li.dropdown.open.active > a:hover,
+.nav > li.dropdown.open.active > a:focus {
+ color: #ffffff;
+ background-color: #999999;
+ border-color: #999999;
+}
+.nav li.dropdown.open .caret,
+.nav li.dropdown.open.active .caret,
+.nav li.dropdown.open a:hover .caret,
+.nav li.dropdown.open a:focus .caret {
+ border-top-color: #ffffff;
+ border-bottom-color: #ffffff;
+ opacity: 1;
+ filter: alpha(opacity=100);
+}
+.tabs-stacked .open > a:hover,
+.tabs-stacked .open > a:focus {
+ border-color: #999999;
+}
+.tabbable {
+ *zoom: 1;
+}
+.tabbable:before,
+.tabbable:after {
+ display: table;
+ content: "";
+ line-height: 0;
+}
+.tabbable:after {
+ clear: both;
+}
+.tab-content {
+ overflow: auto;
+}
+.tabs-below > .nav-tabs,
+.tabs-right > .nav-tabs,
+.tabs-left > .nav-tabs {
+ border-bottom: 0;
+}
+.tab-content > .tab-pane,
+.pill-content > .pill-pane {
+ display: none;
+}
+.tab-content > .active,
+.pill-content > .active {
+ display: block;
+}
+.tabs-below > .nav-tabs {
+ border-top: 1px solid #ddd;
+}
+.tabs-below > .nav-tabs > li {
+ margin-top: -1px;
+ margin-bottom: 0;
+}
+.tabs-below > .nav-tabs > li > a {
+ -webkit-border-radius: 0 0 4px 4px;
+ -moz-border-radius: 0 0 4px 4px;
+ border-radius: 0 0 4px 4px;
+}
+.tabs-below > .nav-tabs > li > a:hover,
+.tabs-below > .nav-tabs > li > a:focus {
+ border-bottom-color: transparent;
+ border-top-color: #ddd;
+}
+.tabs-below > .nav-tabs > .active > a,
+.tabs-below > .nav-tabs > .active > a:hover,
+.tabs-below > .nav-tabs > .active > a:focus {
+ border-color: transparent #ddd #ddd #ddd;
+}
+.tabs-left > .nav-tabs > li,
+.tabs-right > .nav-tabs > li {
+ float: none;
+}
+.tabs-left > .nav-tabs > li > a,
+.tabs-right > .nav-tabs > li > a {
+ min-width: 74px;
+ margin-right: 0;
+ margin-bottom: 3px;
+}
+.tabs-left > .nav-tabs {
+ float: left;
+ margin-right: 19px;
+ border-right: 1px solid #ddd;
+}
+.tabs-left > .nav-tabs > li > a {
+ margin-right: -1px;
+ -webkit-border-radius: 4px 0 0 4px;
+ -moz-border-radius: 4px 0 0 4px;
+ border-radius: 4px 0 0 4px;
+}
+.tabs-left > .nav-tabs > li > a:hover,
+.tabs-left > .nav-tabs > li > a:focus {
+ border-color: #eeeeee #dddddd #eeeeee #eeeeee;
+}
+.tabs-left > .nav-tabs .active > a,
+.tabs-left > .nav-tabs .active > a:hover,
+.tabs-left > .nav-tabs .active > a:focus {
+ border-color: #ddd transparent #ddd #ddd;
+ *border-right-color: #ffffff;
+}
+.tabs-right > .nav-tabs {
+ float: right;
+ margin-left: 19px;
+ border-left: 1px solid #ddd;
+}
+.tabs-right > .nav-tabs > li > a {
+ margin-left: -1px;
+ -webkit-border-radius: 0 4px 4px 0;
+ -moz-border-radius: 0 4px 4px 0;
+ border-radius: 0 4px 4px 0;
+}
+.tabs-right > .nav-tabs > li > a:hover,
+.tabs-right > .nav-tabs > li > a:focus {
+ border-color: #eeeeee #eeeeee #eeeeee #dddddd;
+}
+.tabs-right > .nav-tabs .active > a,
+.tabs-right > .nav-tabs .active > a:hover,
+.tabs-right > .nav-tabs .active > a:focus {
+ border-color: #ddd #ddd #ddd transparent;
+ *border-left-color: #ffffff;
+}
+.nav > .disabled > a {
+ color: #999999;
+}
+.nav > .disabled > a:hover,
+.nav > .disabled > a:focus {
+ text-decoration: none;
+ background-color: transparent;
+ cursor: default;
+}
+.navbar {
+ overflow: visible;
+ margin-bottom: 20px;
+ *position: relative;
+ *z-index: 2;
+}
+.navbar-inner {
+ min-height: 40px;
+ padding-left: 20px;
+ padding-right: 20px;
+ background-color: #fafafa;
+ background-image: -moz-linear-gradient(top, #ffffff, #f2f2f2);
+ background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#ffffff), to(#f2f2f2));
+ background-image: -webkit-linear-gradient(top, #ffffff, #f2f2f2);
+ background-image: -o-linear-gradient(top, #ffffff, #f2f2f2);
+ background-image: linear-gradient(to bottom, #ffffff, #f2f2f2);
+ background-repeat: repeat-x;
+ filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffffffff', endColorstr='#fff2f2f2', GradientType=0);
+ border: 1px solid #d4d4d4;
+ -webkit-border-radius: 4px;
+ -moz-border-radius: 4px;
+ border-radius: 4px;
+ -webkit-box-shadow: 0 1px 4px rgba(0, 0, 0, 0.065);
+ -moz-box-shadow: 0 1px 4px rgba(0, 0, 0, 0.065);
+ box-shadow: 0 1px 4px rgba(0, 0, 0, 0.065);
+ *zoom: 1;
+}
+.navbar-inner:before,
+.navbar-inner:after {
+ display: table;
+ content: "";
+ line-height: 0;
+}
+.navbar-inner:after {
+ clear: both;
+}
+.navbar .container {
+ width: auto;
+}
+.nav-collapse.collapse {
+ height: auto;
+ overflow: visible;
+}
+.navbar .brand {
+ float: left;
+ display: block;
+ padding: 10px 20px 10px;
+ margin-left: -20px;
+ font-size: 20px;
+ font-weight: 200;
+ color: #777777;
+ text-shadow: 0 1px 0 #ffffff;
+}
+.navbar .brand:hover,
+.navbar .brand:focus {
+ text-decoration: none;
+}
+.navbar-text {
+ margin-bottom: 0;
+ line-height: 40px;
+ color: #777777;
+}
+.navbar-link {
+ color: #777777;
+}
+.navbar-link:hover,
+.navbar-link:focus {
+ color: #333333;
+}
+.navbar .divider-vertical {
+ height: 40px;
+ margin: 0 9px;
+ border-left: 1px solid #f2f2f2;
+ border-right: 1px solid #ffffff;
+}
+.navbar .btn,
+.navbar .btn-group {
+ margin-top: 5px;
+}
+.navbar .btn-group .btn,
+.navbar .input-prepend .btn,
+.navbar .input-append .btn,
+.navbar .input-prepend .btn-group,
+.navbar .input-append .btn-group {
+ margin-top: 0;
+}
+.navbar-form {
+ margin-bottom: 0;
+ *zoom: 1;
+}
+.navbar-form:before,
+.navbar-form:after {
+ display: table;
+ content: "";
+ line-height: 0;
+}
+.navbar-form:after {
+ clear: both;
+}
+.navbar-form input,
+.navbar-form select,
+.navbar-form .radio,
+.navbar-form .checkbox {
+ margin-top: 5px;
+}
+.navbar-form input,
+.navbar-form select,
+.navbar-form .btn {
+ display: inline-block;
+ margin-bottom: 0;
+}
+.navbar-form input[type="image"],
+.navbar-form input[type="checkbox"],
+.navbar-form input[type="radio"] {
+ margin-top: 3px;
+}
+.navbar-form .input-append,
+.navbar-form .input-prepend {
+ margin-top: 5px;
+ white-space: nowrap;
+}
+.navbar-form .input-append input,
+.navbar-form .input-prepend input {
+ margin-top: 0;
+}
+.navbar-search {
+ position: relative;
+ float: left;
+ margin-top: 5px;
+ margin-bottom: 0;
+}
+.navbar-search .search-query {
+ margin-bottom: 0;
+ padding: 4px 14px;
+ font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
+ font-size: 13px;
+ font-weight: normal;
+ line-height: 1;
+ -webkit-border-radius: 15px;
+ -moz-border-radius: 15px;
+ border-radius: 15px;
+}
+.navbar-static-top {
+ position: static;
+ margin-bottom: 0;
+}
+.navbar-static-top .navbar-inner {
+ -webkit-border-radius: 0;
+ -moz-border-radius: 0;
+ border-radius: 0;
+}
+.navbar-fixed-top,
+.navbar-fixed-bottom {
+ position: fixed;
+ right: 0;
+ left: 0;
+ z-index: 1030;
+ margin-bottom: 0;
+}
+.navbar-fixed-top .navbar-inner,
+.navbar-static-top .navbar-inner {
+ border-width: 0 0 1px;
+}
+.navbar-fixed-bottom .navbar-inner {
+ border-width: 1px 0 0;
+}
+.navbar-fixed-top .navbar-inner,
+.navbar-fixed-bottom .navbar-inner {
+ padding-left: 0;
+ padding-right: 0;
+ -webkit-border-radius: 0;
+ -moz-border-radius: 0;
+ border-radius: 0;
+}
+.navbar-static-top .container,
+.navbar-fixed-top .container,
+.navbar-fixed-bottom .container {
+ width: 940px;
+}
+.navbar-fixed-top {
+ top: 0;
+}
+.navbar-fixed-top .navbar-inner,
+.navbar-static-top .navbar-inner {
+ -webkit-box-shadow: 0 1px 10px rgba(0,0,0,.1);
+ -moz-box-shadow: 0 1px 10px rgba(0,0,0,.1);
+ box-shadow: 0 1px 10px rgba(0,0,0,.1);
+}
+.navbar-fixed-bottom {
+ bottom: 0;
+}
+.navbar-fixed-bottom .navbar-inner {
+ -webkit-box-shadow: 0 -1px 10px rgba(0,0,0,.1);
+ -moz-box-shadow: 0 -1px 10px rgba(0,0,0,.1);
+ box-shadow: 0 -1px 10px rgba(0,0,0,.1);
+}
+.navbar .nav {
+ position: relative;
+ left: 0;
+ display: block;
+ float: left;
+ margin: 0 10px 0 0;
+}
+.navbar .nav.pull-right {
+ float: right;
+ margin-right: 0;
+}
+.navbar .nav > li {
+ float: left;
+}
+.navbar .nav > li > a {
+ float: none;
+ padding: 10px 15px 10px;
+ color: #777777;
+ text-decoration: none;
+ text-shadow: 0 1px 0 #ffffff;
+}
+.navbar .nav .dropdown-toggle .caret {
+ margin-top: 8px;
+}
+.navbar .nav > li > a:focus,
+.navbar .nav > li > a:hover {
+ background-color: transparent;
+ color: #333333;
+ text-decoration: none;
+}
+.navbar .nav > .active > a,
+.navbar .nav > .active > a:hover,
+.navbar .nav > .active > a:focus {
+ color: #555555;
+ text-decoration: none;
+ background-color: #e5e5e5;
+ -webkit-box-shadow: inset 0 3px 8px rgba(0, 0, 0, 0.125);
+ -moz-box-shadow: inset 0 3px 8px rgba(0, 0, 0, 0.125);
+ box-shadow: inset 0 3px 8px rgba(0, 0, 0, 0.125);
+}
+.navbar .btn-navbar {
+ display: none;
+ float: right;
+ padding: 7px 10px;
+ margin-left: 5px;
+ margin-right: 5px;
+ color: #ffffff;
+ text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25);
+ background-color: #ededed;
+ background-image: -moz-linear-gradient(top, #f2f2f2, #e5e5e5);
+ background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#f2f2f2), to(#e5e5e5));
+ background-image: -webkit-linear-gradient(top, #f2f2f2, #e5e5e5);
+ background-image: -o-linear-gradient(top, #f2f2f2, #e5e5e5);
+ background-image: linear-gradient(to bottom, #f2f2f2, #e5e5e5);
+ background-repeat: repeat-x;
+ filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#fff2f2f2', endColorstr='#ffe5e5e5', GradientType=0);
+ border-color: #e5e5e5 #e5e5e5 #bfbfbf;
+ border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
+ *background-color: #e5e5e5;
+ /* Darken IE7 buttons by default so they stand out more given they won't have borders */
+
+ filter: progid:DXImageTransform.Microsoft.gradient(enabled = false);
+ -webkit-box-shadow: inset 0 1px 0 rgba(255,255,255,.1), 0 1px 0 rgba(255,255,255,.075);
+ -moz-box-shadow: inset 0 1px 0 rgba(255,255,255,.1), 0 1px 0 rgba(255,255,255,.075);
+ box-shadow: inset 0 1px 0 rgba(255,255,255,.1), 0 1px 0 rgba(255,255,255,.075);
+}
+.navbar .btn-navbar:hover,
+.navbar .btn-navbar:focus,
+.navbar .btn-navbar:active,
+.navbar .btn-navbar.active,
+.navbar .btn-navbar.disabled,
+.navbar .btn-navbar[disabled] {
+ color: #ffffff;
+ background-color: #e5e5e5;
+ *background-color: #d9d9d9;
+}
+.navbar .btn-navbar:active,
+.navbar .btn-navbar.active {
+ background-color: #cccccc \9;
+}
+.navbar .btn-navbar .icon-bar {
+ display: block;
+ width: 18px;
+ height: 2px;
+ background-color: #f5f5f5;
+ -webkit-border-radius: 1px;
+ -moz-border-radius: 1px;
+ border-radius: 1px;
+ -webkit-box-shadow: 0 1px 0 rgba(0, 0, 0, 0.25);
+ -moz-box-shadow: 0 1px 0 rgba(0, 0, 0, 0.25);
+ box-shadow: 0 1px 0 rgba(0, 0, 0, 0.25);
+}
+.btn-navbar .icon-bar + .icon-bar {
+ margin-top: 3px;
+}
+.navbar .nav > li > .dropdown-menu:before {
+ content: '';
+ display: inline-block;
+ border-left: 7px solid transparent;
+ border-right: 7px solid transparent;
+ border-bottom: 7px solid #ccc;
+ border-bottom-color: rgba(0, 0, 0, 0.2);
+ position: absolute;
+ top: -7px;
+ left: 9px;
+}
+.navbar .nav > li > .dropdown-menu:after {
+ content: '';
+ display: inline-block;
+ border-left: 6px solid transparent;
+ border-right: 6px solid transparent;
+ border-bottom: 6px solid #ffffff;
+ position: absolute;
+ top: -6px;
+ left: 10px;
+}
+.navbar-fixed-bottom .nav > li > .dropdown-menu:before {
+ border-top: 7px solid #ccc;
+ border-top-color: rgba(0, 0, 0, 0.2);
+ border-bottom: 0;
+ bottom: -7px;
+ top: auto;
+}
+.navbar-fixed-bottom .nav > li > .dropdown-menu:after {
+ border-top: 6px solid #ffffff;
+ border-bottom: 0;
+ bottom: -6px;
+ top: auto;
+}
+.navbar .nav li.dropdown > a:hover .caret,
+.navbar .nav li.dropdown > a:focus .caret {
+ border-top-color: #333333;
+ border-bottom-color: #333333;
+}
+.navbar .nav li.dropdown.open > .dropdown-toggle,
+.navbar .nav li.dropdown.active > .dropdown-toggle,
+.navbar .nav li.dropdown.open.active > .dropdown-toggle {
+ background-color: #e5e5e5;
+ color: #555555;
+}
+.navbar .nav li.dropdown > .dropdown-toggle .caret {
+ border-top-color: #777777;
+ border-bottom-color: #777777;
+}
+.navbar .nav li.dropdown.open > .dropdown-toggle .caret,
+.navbar .nav li.dropdown.active > .dropdown-toggle .caret,
+.navbar .nav li.dropdown.open.active > .dropdown-toggle .caret {
+ border-top-color: #555555;
+ border-bottom-color: #555555;
+}
+.navbar .pull-right > li > .dropdown-menu,
+.navbar .nav > li > .dropdown-menu.pull-right {
+ left: auto;
+ right: 0;
+}
+.navbar .pull-right > li > .dropdown-menu:before,
+.navbar .nav > li > .dropdown-menu.pull-right:before {
+ left: auto;
+ right: 12px;
+}
+.navbar .pull-right > li > .dropdown-menu:after,
+.navbar .nav > li > .dropdown-menu.pull-right:after {
+ left: auto;
+ right: 13px;
+}
+.navbar .pull-right > li > .dropdown-menu .dropdown-menu,
+.navbar .nav > li > .dropdown-menu.pull-right .dropdown-menu {
+ left: auto;
+ right: 100%;
+ margin-left: 0;
+ margin-right: -1px;
+ -webkit-border-radius: 6px 0 6px 6px;
+ -moz-border-radius: 6px 0 6px 6px;
+ border-radius: 6px 0 6px 6px;
+}
+.navbar-inverse .navbar-inner {
+ background-color: #1b1b1b;
+ background-image: -moz-linear-gradient(top, #222222, #111111);
+ background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#222222), to(#111111));
+ background-image: -webkit-linear-gradient(top, #222222, #111111);
+ background-image: -o-linear-gradient(top, #222222, #111111);
+ background-image: linear-gradient(to bottom, #222222, #111111);
+ background-repeat: repeat-x;
+ filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff222222', endColorstr='#ff111111', GradientType=0);
+ border-color: #252525;
+}
+.navbar-inverse .brand,
+.navbar-inverse .nav > li > a {
+ color: #999999;
+ text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25);
+}
+.navbar-inverse .brand:hover,
+.navbar-inverse .nav > li > a:hover,
+.navbar-inverse .brand:focus,
+.navbar-inverse .nav > li > a:focus {
+ color: #ffffff;
+}
+.navbar-inverse .brand {
+ color: #999999;
+}
+.navbar-inverse .navbar-text {
+ color: #999999;
+}
+.navbar-inverse .nav > li > a:focus,
+.navbar-inverse .nav > li > a:hover {
+ background-color: transparent;
+ color: #ffffff;
+}
+.navbar-inverse .nav .active > a,
+.navbar-inverse .nav .active > a:hover,
+.navbar-inverse .nav .active > a:focus {
+ color: #ffffff;
+ background-color: #111111;
+}
+.navbar-inverse .navbar-link {
+ color: #999999;
+}
+.navbar-inverse .navbar-link:hover,
+.navbar-inverse .navbar-link:focus {
+ color: #ffffff;
+}
+.navbar-inverse .divider-vertical {
+ border-left-color: #111111;
+ border-right-color: #222222;
+}
+.navbar-inverse .nav li.dropdown.open > .dropdown-toggle,
+.navbar-inverse .nav li.dropdown.active > .dropdown-toggle,
+.navbar-inverse .nav li.dropdown.open.active > .dropdown-toggle {
+ background-color: #111111;
+ color: #ffffff;
+}
+.navbar-inverse .nav li.dropdown > a:hover .caret,
+.navbar-inverse .nav li.dropdown > a:focus .caret {
+ border-top-color: #ffffff;
+ border-bottom-color: #ffffff;
+}
+.navbar-inverse .nav li.dropdown > .dropdown-toggle .caret {
+ border-top-color: #999999;
+ border-bottom-color: #999999;
+}
+.navbar-inverse .nav li.dropdown.open > .dropdown-toggle .caret,
+.navbar-inverse .nav li.dropdown.active > .dropdown-toggle .caret,
+.navbar-inverse .nav li.dropdown.open.active > .dropdown-toggle .caret {
+ border-top-color: #ffffff;
+ border-bottom-color: #ffffff;
+}
+.navbar-inverse .navbar-search .search-query {
+ color: #ffffff;
+ background-color: #515151;
+ border-color: #111111;
+ -webkit-box-shadow: inset 0 1px 2px rgba(0,0,0,.1), 0 1px 0 rgba(255,255,255,.15);
+ -moz-box-shadow: inset 0 1px 2px rgba(0,0,0,.1), 0 1px 0 rgba(255,255,255,.15);
+ box-shadow: inset 0 1px 2px rgba(0,0,0,.1), 0 1px 0 rgba(255,255,255,.15);
+ -webkit-transition: none;
+ -moz-transition: none;
+ -o-transition: none;
+ transition: none;
+}
+.navbar-inverse .navbar-search .search-query:-moz-placeholder {
+ color: #cccccc;
+}
+.navbar-inverse .navbar-search .search-query:-ms-input-placeholder {
+ color: #cccccc;
+}
+.navbar-inverse .navbar-search .search-query::-webkit-input-placeholder {
+ color: #cccccc;
+}
+.navbar-inverse .navbar-search .search-query:focus,
+.navbar-inverse .navbar-search .search-query.focused {
+ padding: 5px 15px;
+ color: #333333;
+ text-shadow: 0 1px 0 #ffffff;
+ background-color: #ffffff;
+ border: 0;
+ -webkit-box-shadow: 0 0 3px rgba(0, 0, 0, 0.15);
+ -moz-box-shadow: 0 0 3px rgba(0, 0, 0, 0.15);
+ box-shadow: 0 0 3px rgba(0, 0, 0, 0.15);
+ outline: 0;
+}
+.navbar-inverse .btn-navbar {
+ color: #ffffff;
+ text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25);
+ background-color: #0e0e0e;
+ background-image: -moz-linear-gradient(top, #151515, #040404);
+ background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#151515), to(#040404));
+ background-image: -webkit-linear-gradient(top, #151515, #040404);
+ background-image: -o-linear-gradient(top, #151515, #040404);
+ background-image: linear-gradient(to bottom, #151515, #040404);
+ background-repeat: repeat-x;
+ filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff151515', endColorstr='#ff040404', GradientType=0);
+ border-color: #040404 #040404 #000000;
+ border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
+ *background-color: #040404;
+ /* Darken IE7 buttons by default so they stand out more given they won't have borders */
+
+ filter: progid:DXImageTransform.Microsoft.gradient(enabled = false);
+}
+.navbar-inverse .btn-navbar:hover,
+.navbar-inverse .btn-navbar:focus,
+.navbar-inverse .btn-navbar:active,
+.navbar-inverse .btn-navbar.active,
+.navbar-inverse .btn-navbar.disabled,
+.navbar-inverse .btn-navbar[disabled] {
+ color: #ffffff;
+ background-color: #040404;
+ *background-color: #000000;
+}
+.navbar-inverse .btn-navbar:active,
+.navbar-inverse .btn-navbar.active {
+ background-color: #000000 \9;
+}
+/*@import "bootstrap/breadcrumbs.less";*/
+/*@import "bootstrap/pagination.less";*/
+/*@import "bootstrap/pager.less";*/
+.modal-backdrop {
+ position: fixed;
+ top: 0;
+ right: 0;
+ bottom: 0;
+ left: 0;
+ z-index: 1040;
+ background-color: #000000;
+}
+.modal-backdrop.fade {
+ opacity: 0;
+}
+.modal-backdrop,
+.modal-backdrop.fade.in {
+ opacity: 0.8;
+ filter: alpha(opacity=80);
+}
+.modal {
+ position: fixed;
+ top: 10%;
+ left: 50%;
+ z-index: 1050;
+ width: 560px;
+ margin-left: -280px;
+ background-color: #ffffff;
+ border: 1px solid #999;
+ border: 1px solid rgba(0, 0, 0, 0.3);
+ *border: 1px solid #999;
+ /* IE6-7 */
+
+ -webkit-border-radius: 6px;
+ -moz-border-radius: 6px;
+ border-radius: 6px;
+ -webkit-box-shadow: 0 3px 7px rgba(0, 0, 0, 0.3);
+ -moz-box-shadow: 0 3px 7px rgba(0, 0, 0, 0.3);
+ box-shadow: 0 3px 7px rgba(0, 0, 0, 0.3);
+ -webkit-background-clip: padding-box;
+ -moz-background-clip: padding-box;
+ background-clip: padding-box;
+ outline: none;
+}
+.modal.fade {
+ -webkit-transition: opacity .3s linear, top .3s ease-out;
+ -moz-transition: opacity .3s linear, top .3s ease-out;
+ -o-transition: opacity .3s linear, top .3s ease-out;
+ transition: opacity .3s linear, top .3s ease-out;
+ top: -25%;
+}
+.modal.fade.in {
+ top: 10%;
+}
+.modal-header {
+ padding: 9px 15px;
+ border-bottom: 1px solid #eee;
+}
+.modal-header .close {
+ margin-top: 2px;
+}
+.modal-header h3 {
+ margin: 0;
+ line-height: 30px;
+}
+.modal-body {
+ position: relative;
+ overflow-y: auto;
+ max-height: 400px;
+ padding: 15px;
+}
+.modal-form {
+ margin-bottom: 0;
+}
+.modal-footer {
+ padding: 14px 15px 15px;
+ margin-bottom: 0;
+ text-align: right;
+ background-color: #f5f5f5;
+ border-top: 1px solid #ddd;
+ -webkit-border-radius: 0 0 6px 6px;
+ -moz-border-radius: 0 0 6px 6px;
+ border-radius: 0 0 6px 6px;
+ -webkit-box-shadow: inset 0 1px 0 #ffffff;
+ -moz-box-shadow: inset 0 1px 0 #ffffff;
+ box-shadow: inset 0 1px 0 #ffffff;
+ *zoom: 1;
+}
+.modal-footer:before,
+.modal-footer:after {
+ display: table;
+ content: "";
+ line-height: 0;
+}
+.modal-footer:after {
+ clear: both;
+}
+.modal-footer .btn + .btn {
+ margin-left: 5px;
+ margin-bottom: 0;
+}
+.modal-footer .btn-group .btn + .btn {
+ margin-left: -1px;
+}
+.modal-footer .btn-block + .btn-block {
+ margin-left: 0;
+}
+/*@import "bootstrap/tooltip.less";*/
+/*@import "bootstrap/popovers.less";*/
+/*@import "bootstrap/thumbnails.less";*/
+/*@import "bootstrap/media.less";*/
+/*@import "bootstrap/labels-badges.less";*/
+/*@import "bootstrap/progress-bars.less";*/
+/*@import "bootstrap/accordion.less";*/
+/*@import "bootstrap/carousel.less";*/
+/*@import "bootstrap/hero-unit.less";*/
+.pull-right {
+ float: right;
+}
+.pull-left {
+ float: left;
+}
+.hide {
+ display: none;
+}
+.show {
+ display: block;
+}
+.invisible {
+ visibility: hidden;
+}
+.affix {
+ position: fixed;
+}
+/* http://meyerweb.com/eric/tools/css/reset/
+ v2.0 | 20110126
+ License: none (public domain)
+*/
+html,
+body,
+div,
+span,
+applet,
+object,
+iframe,
+h1,
+h2,
+h3,
+h4,
+h5,
+h6,
+p,
+blockquote,
+pre,
+a,
+abbr,
+acronym,
+address,
+big,
+cite,
+code,
+del,
+dfn,
+em,
+img,
+ins,
+kbd,
+q,
+s,
+samp,
+small,
+strike,
+strong,
+sub,
+sup,
+tt,
+var,
+b,
+u,
+i,
+center,
+dl,
+dt,
+dd,
+ol,
+ul,
+li,
+fieldset,
+form,
+label,
+legend,
+table,
+caption,
+tbody,
+tfoot,
+thead,
+tr,
+th,
+td,
+article,
+aside,
+canvas,
+details,
+embed,
+figure,
+figcaption,
+footer,
+header,
+hgroup,
+menu,
+nav,
+output,
+ruby,
+section,
+summary,
+time,
+mark,
+audio,
+video {
+ margin: 0;
+ padding: 0;
+ border: 0;
+ font-size: 100%;
+ font: inherit;
+ vertical-align: baseline;
+}
+/* HTML5 display-role reset for older browsers */
+article,
+asid
<TRUNCATED>
[05/30] apex-malhar git commit: Renamed demos to examples. Packages
and artifactid names are changed as suggested.
Posted by th...@apache.org.
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/OutputCollectorImpl.java
----------------------------------------------------------------------
diff --git a/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/OutputCollectorImpl.java b/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/OutputCollectorImpl.java
new file mode 100644
index 0000000..31337c1
--- /dev/null
+++ b/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/OutputCollectorImpl.java
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.mroperator;
+
+import java.io.IOException;
+import java.io.PipedInputStream;
+import java.io.PipedOutputStream;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.serializer.Deserializer;
+import org.apache.hadoop.io.serializer.SerializationFactory;
+import org.apache.hadoop.io.serializer.Serializer;
+import org.apache.hadoop.mapred.OutputCollector;
+import com.datatorrent.lib.util.KeyHashValPair;
+
+/**
+ * <p>OutputCollectorImpl class.</p>
+ *
+ * @since 0.9.0
+ */
+@SuppressWarnings("unchecked")
+public class OutputCollectorImpl<K extends Object, V extends Object> implements OutputCollector<K, V>
+{
+ private static final Logger logger = LoggerFactory.getLogger(OutputCollectorImpl.class);
+
+ private List<KeyHashValPair<K, V>> list = new ArrayList<KeyHashValPair<K, V>>();
+
+ public List<KeyHashValPair<K, V>> getList()
+ {
+ return list;
+ }
+
+ private transient SerializationFactory serializationFactory;
+ private transient Configuration conf = null;
+
+ public OutputCollectorImpl()
+ {
+ conf = new Configuration();
+ serializationFactory = new SerializationFactory(conf);
+
+ }
+
+ private <T> T cloneObj(T t) throws IOException
+ {
+ Serializer<T> keySerializer;
+ Class<T> keyClass;
+ PipedInputStream pis = new PipedInputStream();
+ PipedOutputStream pos = new PipedOutputStream(pis);
+ keyClass = (Class<T>)t.getClass();
+ keySerializer = serializationFactory.getSerializer(keyClass);
+ keySerializer.open(pos);
+ keySerializer.serialize(t);
+ Deserializer<T> keyDesiralizer = serializationFactory.getDeserializer(keyClass);
+ keyDesiralizer.open(pis);
+ T clonedArg0 = keyDesiralizer.deserialize(null);
+ pos.close();
+ pis.close();
+ keySerializer.close();
+ keyDesiralizer.close();
+ return clonedArg0;
+
+ }
+
+ @Override
+ public void collect(K arg0, V arg1) throws IOException
+ {
+ if (conf == null) {
+ conf = new Configuration();
+ serializationFactory = new SerializationFactory(conf);
+ }
+ list.add(new KeyHashValPair<K, V>(cloneObj(arg0), cloneObj(arg1)));
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/ReduceOperator.java
----------------------------------------------------------------------
diff --git a/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/ReduceOperator.java b/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/ReduceOperator.java
new file mode 100644
index 0000000..cdc5ec9
--- /dev/null
+++ b/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/ReduceOperator.java
@@ -0,0 +1,188 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.mroperator;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapred.Counters;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reducer;
+import org.apache.hadoop.mapred.Reporter;
+
+import com.datatorrent.api.Context.OperatorContext;
+import com.datatorrent.api.DefaultInputPort;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.Operator;
+import com.datatorrent.lib.util.KeyHashValPair;
+
+/**
+ * <p>ReduceOperator class.</p>
+ *
+ * @since 0.9.0
+ */
+@SuppressWarnings({ "deprecation", "unused" })
+public class ReduceOperator<K1, V1, K2, V2> implements Operator
+{
+ private static final Logger logger = LoggerFactory.getLogger(ReduceOperator.class);
+
+ private Class<? extends Reducer<K1, V1, K2, V2>> reduceClass;
+ private transient Reducer<K1, V1, K2, V2> reduceObj;
+ private transient Reporter reporter;
+ private OutputCollector<K2, V2> outputCollector;
+ private String configFile;
+
+ public Class<? extends Reducer<K1, V1, K2, V2>> getReduceClass()
+ {
+ return reduceClass;
+ }
+
+ public void setReduceClass(Class<? extends Reducer<K1, V1, K2, V2>> reduceClass)
+ {
+ this.reduceClass = reduceClass;
+ }
+
+ public String getConfigFile()
+ {
+ return configFile;
+ }
+
+ public void setConfigFile(String configFile)
+ {
+ this.configFile = configFile;
+ }
+
+ private int numberOfMappersRunning = -1;
+ private int operatorId;
+
+ public final transient DefaultInputPort<KeyHashValPair<Integer, Integer>> inputCount = new DefaultInputPort<KeyHashValPair<Integer, Integer>>()
+ {
+ @Override
+ public void process(KeyHashValPair<Integer, Integer> tuple)
+ {
+ logger.info("processing {}", tuple);
+ if (numberOfMappersRunning == -1) {
+ numberOfMappersRunning = tuple.getValue();
+ } else {
+ numberOfMappersRunning += tuple.getValue();
+ }
+
+ }
+
+ };
+
+ public final transient DefaultOutputPort<KeyHashValPair<K2, V2>> output = new DefaultOutputPort<KeyHashValPair<K2, V2>>();
+ private Map<K1, List<V1>> cacheObject;
+ public final transient DefaultInputPort<KeyHashValPair<K1, V1>> input = new DefaultInputPort<KeyHashValPair<K1, V1>>()
+ {
+ @Override
+ public void process(KeyHashValPair<K1, V1> tuple)
+ {
+ // logger.info("processing tupple {}",tuple);
+ List<V1> list = cacheObject.get(tuple.getKey());
+ if (list == null) {
+ list = new ArrayList<V1>();
+ list.add(tuple.getValue());
+ cacheObject.put(tuple.getKey(), list);
+ } else {
+ list.add(tuple.getValue());
+ }
+ }
+
+ };
+
+ @Override
+ public void setup(OperatorContext context)
+ {
+ reporter = new ReporterImpl(ReporterImpl.ReporterType.Reducer, new Counters());
+ if (context != null) {
+ operatorId = context.getId();
+ }
+ cacheObject = new HashMap<K1, List<V1>>();
+ outputCollector = new OutputCollectorImpl<K2, V2>();
+ if (reduceClass != null) {
+ try {
+ reduceObj = reduceClass.newInstance();
+ } catch (Exception e) {
+ logger.info("can't instantiate object {}", e.getMessage());
+ throw new RuntimeException(e);
+ }
+ Configuration conf = new Configuration();
+ InputStream stream = null;
+ if (configFile != null && configFile.length() > 0) {
+ logger.info("system /{}", configFile);
+ stream = ClassLoader.getSystemResourceAsStream("/" + configFile);
+ if (stream == null) {
+ logger.info("system {}", configFile);
+ stream = ClassLoader.getSystemResourceAsStream(configFile);
+ }
+ }
+ if (stream != null) {
+ logger.info("found our stream... so adding it");
+ conf.addResource(stream);
+ }
+ reduceObj.configure(new JobConf(conf));
+ }
+
+ }
+
+ @Override
+ public void teardown()
+ {
+
+ }
+
+ @Override
+ public void beginWindow(long windowId)
+ {
+
+ }
+
+ @Override
+ public void endWindow()
+ {
+ if (numberOfMappersRunning == 0) {
+ for (Map.Entry<K1, List<V1>> e : cacheObject.entrySet()) {
+ try {
+ reduceObj.reduce(e.getKey(), e.getValue().iterator(), outputCollector, reporter);
+ } catch (IOException e1) {
+ logger.info(e1.getMessage());
+ throw new RuntimeException(e1);
+ }
+ }
+ List<KeyHashValPair<K2, V2>> list = ((OutputCollectorImpl<K2, V2>)outputCollector).getList();
+ for (KeyHashValPair<K2, V2> e : list) {
+ output.emit(e);
+ }
+ list.clear();
+ cacheObject.clear();
+ numberOfMappersRunning = -1;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/ReporterImpl.java
----------------------------------------------------------------------
diff --git a/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/ReporterImpl.java b/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/ReporterImpl.java
new file mode 100644
index 0000000..cfbb26e
--- /dev/null
+++ b/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/ReporterImpl.java
@@ -0,0 +1,119 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.mroperator;
+
+import org.apache.hadoop.mapred.Counters;
+import org.apache.hadoop.mapred.Counters.Counter;
+import org.apache.hadoop.mapred.InputSplit;
+import org.apache.hadoop.mapred.Reporter;
+
+/**
+ * <p>ReporterImpl class.</p>
+ *
+ * @since 0.9.0
+ */
+public class ReporterImpl implements Reporter
+{
+ private Counters counters;
+ InputSplit inputSplit;
+
+ public enum ReporterType
+ {
+ Mapper, Reducer
+ }
+
+ private ReporterType typ;
+
+ public ReporterImpl(final ReporterType kind, final Counters ctrs)
+ {
+ this.typ = kind;
+ this.counters = ctrs;
+ }
+
+ @Override
+ public InputSplit getInputSplit()
+ {
+ if (typ == ReporterType.Reducer) {
+ throw new UnsupportedOperationException("Reducer cannot call getInputSplit()");
+ } else {
+ return inputSplit;
+ }
+ }
+
+ public void setInputSplit(InputSplit inputSplit)
+ {
+ this.inputSplit = inputSplit;
+ }
+
+ @Override
+ public void incrCounter(Enum<?> key, long amount)
+ {
+ if (null != counters) {
+ counters.incrCounter(key, amount);
+ }
+ }
+
+ @Override
+ public void incrCounter(String group, String counter, long amount)
+ {
+ if (null != counters) {
+ counters.incrCounter(group, counter, amount);
+ }
+ }
+
+ @Override
+ public void setStatus(String status)
+ {
+ // do nothing.
+ }
+
+ @Override
+ public void progress()
+ {
+ // do nothing.
+ }
+
+ @Override
+ public Counter getCounter(String group, String name)
+ {
+ Counters.Counter counter = null;
+ if (counters != null) {
+ counter = counters.findCounter(group, name);
+ }
+
+ return counter;
+ }
+
+ @Override
+ public Counter getCounter(Enum<?> key)
+ {
+ Counters.Counter counter = null;
+ if (counters != null) {
+ counter = counters.findCounter(key);
+ }
+
+ return counter;
+ }
+
+ public float getProgress()
+ {
+ return 0;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/WordCount.java
----------------------------------------------------------------------
diff --git a/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/WordCount.java b/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/WordCount.java
new file mode 100644
index 0000000..31ce3a9
--- /dev/null
+++ b/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/WordCount.java
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.mroperator;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.StringTokenizer;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.FileOutputFormat;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.MapReduceBase;
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reducer;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.TextInputFormat;
+import org.apache.hadoop.mapred.TextOutputFormat;
+
+/**
+ * <p>WordCount class.</p>
+ *
+ * @since 0.9.0
+ */
+@SuppressWarnings("deprecation")
+public class WordCount
+{
+
+ public static class Map extends MapReduceBase implements Mapper<LongWritable, Text, Text, IntWritable>
+ {
+ private static final IntWritable one = new IntWritable(1);
+ private Text word = new Text();
+
+ public void map(LongWritable key, Text value, OutputCollector<Text, IntWritable> output, Reporter reporter) throws IOException
+ {
+ String line = value.toString();
+ StringTokenizer tokenizer = new StringTokenizer(line);
+ while (tokenizer.hasMoreTokens()) {
+ word.set(tokenizer.nextToken());
+ output.collect(word, one);
+ }
+ }
+ }
+
+ public static class Reduce extends MapReduceBase implements Reducer<Text, IntWritable, Text, IntWritable>
+ {
+ public void reduce(Text key, Iterator<IntWritable> values, OutputCollector<Text, IntWritable> output, Reporter reporter) throws IOException
+ {
+ int sum = 0;
+ while (values.hasNext()) {
+ sum += values.next().get();
+ }
+ output.collect(key, new IntWritable(sum));
+ }
+ }
+
+ public void run(String[] args) throws Exception
+ {
+
+ JobConf conf = new JobConf(this.getClass());
+ conf.setJobName("wordcount");
+
+ conf.setOutputKeyClass(Text.class);
+ conf.setOutputValueClass(IntWritable.class);
+
+ conf.setMapperClass(Map.class);
+ conf.setCombinerClass(Reduce.class);
+ conf.setReducerClass(Reduce.class);
+
+ conf.setInputFormat(TextInputFormat.class);
+ conf.setOutputFormat(TextOutputFormat.class);
+
+ FileInputFormat.setInputPaths(conf, new Path(args[0]));
+ FileOutputFormat.setOutputPath(conf, new Path(args[1]));
+
+ JobClient.runJob(conf);
+ }
+
+ public static void main(String[] args) throws Exception
+ {
+ new WordCount().run(args);
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/mroperator/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/examples/mroperator/src/main/resources/META-INF/properties.xml b/examples/mroperator/src/main/resources/META-INF/properties.xml
new file mode 100644
index 0000000..5a56014
--- /dev/null
+++ b/examples/mroperator/src/main/resources/META-INF/properties.xml
@@ -0,0 +1,88 @@
+<?xml version="1.0"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<configuration>
+ <!-- Log Count Example -->
+ <property>
+ <name>dt.application.LogsCountExample.operator.Mapper.dirName</name>
+ <value></value>
+ </property>
+ <property>
+ <name>dt.application.LogsCountExample.operator.Mapper.partitionCount</name>
+ <value></value>
+ </property>
+ <property>
+ <name>dt.application.LogsCountExample.operator.Console.filePath</name>
+ <value></value>
+ </property>
+ <property>
+ <name>dt.application.LogsCountExample.operator.Console.outputFileName</name>
+ <value></value>
+ </property>
+ <property>
+ <name>dt.application.LogsCountExample.operator.Reducer.attr.PARTITIONER</name>
+ <value>com.datatorrent.common.partitioner.StatelessPartitioner:1</value>
+ </property>
+
+ <!-- Word Count Example -->
+ <property>
+ <name>dt.application.WordCountExample.operator.Mapper.dirName</name>
+ <value></value>
+ </property>
+ <property>
+ <name>dt.application.WordCountExample.operator.Mapper.partitionCount</name>
+ <value></value>
+ </property>
+ <property>
+ <name>dt.application.WordCountExample.operator.Console.filePath</name>
+ <value></value>
+ </property>
+ <property>
+ <name>dt.application.WordCountExample.operator.Console.outputFileName</name>
+ <value></value>
+ </property>
+ <property>
+ <name>dt.application.WordCountExample.operator.Reducer.attr.PARTITIONER</name>
+ <value>com.datatorrent.common.partitioner.StatelessPartitioner:1</value>
+ </property>
+
+ <!-- Inverted Index Example -->
+ <property>
+ <name>dt.application.InvertedIndexExample.operator.Mapper.dirName</name>
+ <value></value>
+ </property>
+ <property>
+ <name>dt.application.InvertedIndexExample.operator.Mapper.partitionCount</name>
+ <value></value>
+ </property>
+ <property>
+ <name>dt.application.InvertedIndexExample.operator.Console.filePath</name>
+ <value></value>
+ </property>
+ <property>
+ <name>dt.application.InvertedIndexExample.operator.Console.outputFileName</name>
+ <value></value>
+ </property>
+ <property>
+ <name>dt.application.LogsCountExample.operator.Reducer.attr.PARTITIONER</name>
+ <value>com.datatorrent.common.partitioner.StatelessPartitioner:1</value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/mroperator/src/test/java/org/apache/apex/examples/mroperator/MapOperatorTest.java
----------------------------------------------------------------------
diff --git a/examples/mroperator/src/test/java/org/apache/apex/examples/mroperator/MapOperatorTest.java b/examples/mroperator/src/test/java/org/apache/apex/examples/mroperator/MapOperatorTest.java
new file mode 100644
index 0000000..073d847
--- /dev/null
+++ b/examples/mroperator/src/test/java/org/apache/apex/examples/mroperator/MapOperatorTest.java
@@ -0,0 +1,159 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.mroperator;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestWatcher;
+import org.junit.runner.Description;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.serializer.SerializationFactory;
+import org.apache.hadoop.io.serializer.Serializer;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.InputSplit;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.TextInputFormat;
+
+import com.datatorrent.lib.testbench.CollectorTestSink;
+
+@SuppressWarnings("deprecation")
+public class MapOperatorTest
+{
+
+ private static Logger LOG = LoggerFactory.getLogger(MapOperatorTest.class);
+
+ @Rule
+ public TestMeta testMeta = new TestMeta();
+ /**
+ * Test node logic emits correct results
+ */
+ @Test
+ public void testNodeProcessing() throws Exception
+ {
+ testNodeProcessingSchema(new MapOperator<LongWritable, Text, Text, IntWritable>());
+ }
+
+ public void testNodeProcessingSchema(MapOperator<LongWritable, Text, Text, IntWritable> oper) throws IOException
+ {
+
+ CollectorTestSink sortSink = new CollectorTestSink();
+ oper.output.setSink(sortSink);
+
+ oper.setMapClass(WordCount.Map.class);
+ oper.setCombineClass(WordCount.Reduce.class);
+ oper.setDirName(testMeta.testDir);
+ oper.setConfigFile(null);
+ oper.setInputFormatClass(TextInputFormat.class);
+
+ Configuration conf = new Configuration();
+ JobConf jobConf = new JobConf(conf);
+ FileInputFormat.setInputPaths(jobConf, new Path(testMeta.testDir));
+ TextInputFormat inputFormat = new TextInputFormat();
+ inputFormat.configure(jobConf);
+ InputSplit[] splits = inputFormat.getSplits(jobConf, 1);
+ SerializationFactory serializationFactory = new SerializationFactory(conf);
+ Serializer keySerializer = serializationFactory.getSerializer(splits[0].getClass());
+ keySerializer.open(oper.getOutstream());
+ keySerializer.serialize(splits[0]);
+ oper.setInputSplitClass(splits[0].getClass());
+ keySerializer.close();
+ oper.setup(null);
+ oper.beginWindow(0);
+ oper.emitTuples();
+ oper.emitTuples();
+ oper.endWindow();
+ oper.beginWindow(1);
+ oper.emitTuples();
+ oper.endWindow();
+
+ Assert.assertEquals("number emitted tuples", 3, sortSink.collectedTuples.size());
+ for (Object o : sortSink.collectedTuples) {
+ LOG.debug(o.toString());
+ }
+ LOG.debug("Done testing round\n");
+ oper.teardown();
+ }
+
+ public static class TestMeta extends TestWatcher
+ {
+ public final String file1 = "file1";
+ public String baseDir;
+ public String testDir;
+
+ @Override
+ protected void starting(org.junit.runner.Description description)
+ {
+ String methodName = description.getMethodName();
+ String className = description.getClassName();
+ baseDir = "target/" + className;
+ testDir = baseDir + "/" + methodName;
+ try {
+ FileUtils.forceMkdir(new File(testDir));
+ } catch (IOException ex) {
+ throw new RuntimeException(ex);
+ }
+ createFile(testDir + "/" + file1, "1\n2\n3\n1\n2\n3\n");
+ }
+
+ private void createFile(String fileName, String data)
+ {
+ BufferedWriter output = null;
+ try {
+ output = new BufferedWriter(new FileWriter(new File(fileName)));
+ output.write(data);
+ } catch (IOException ex) {
+ throw new RuntimeException(ex);
+ } finally {
+ if (output != null) {
+ try {
+ output.close();
+ } catch (IOException ex) {
+ LOG.error("not able to close the output stream: ", ex);
+ }
+ }
+ }
+ }
+
+ @Override
+ protected void finished(Description description)
+ {
+ try {
+ FileUtils.deleteDirectory(new File(baseDir));
+ } catch (IOException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/mroperator/src/test/java/org/apache/apex/examples/mroperator/ReduceOperatorTest.java
----------------------------------------------------------------------
diff --git a/examples/mroperator/src/test/java/org/apache/apex/examples/mroperator/ReduceOperatorTest.java b/examples/mroperator/src/test/java/org/apache/apex/examples/mroperator/ReduceOperatorTest.java
new file mode 100644
index 0000000..bff982a
--- /dev/null
+++ b/examples/mroperator/src/test/java/org/apache/apex/examples/mroperator/ReduceOperatorTest.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.mroperator;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+
+import com.datatorrent.lib.testbench.CollectorTestSink;
+import com.datatorrent.lib.util.KeyHashValPair;
+
+public class ReduceOperatorTest
+{
+ private static Logger logger = LoggerFactory.getLogger(ReduceOperatorTest.class);
+
+ /**
+ * Test node logic emits correct results
+ */
+ @Test
+ public void testNodeProcessing() throws Exception
+ {
+ testNodeProcessingSchema(new ReduceOperator<Text, IntWritable,Text, IntWritable>());
+ }
+
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ public void testNodeProcessingSchema(ReduceOperator<Text, IntWritable,Text, IntWritable> oper)
+ {
+ oper.setReduceClass(WordCount.Reduce.class);
+ oper.setConfigFile(null);
+ oper.setup(null);
+
+ CollectorTestSink sortSink = new CollectorTestSink();
+ oper.output.setSink(sortSink);
+
+ oper.beginWindow(0);
+ oper.inputCount.process(new KeyHashValPair<Integer, Integer>(1, 1));
+ oper.input.process(new KeyHashValPair<Text, IntWritable>(new Text("one"), new IntWritable(1)));
+ oper.input.process(new KeyHashValPair<Text, IntWritable>(new Text("one"), new IntWritable(1)));
+ oper.input.process(new KeyHashValPair<Text, IntWritable>(new Text("two"), new IntWritable(1)));
+ oper.endWindow();
+
+ oper.beginWindow(1);
+ oper.input.process(new KeyHashValPair<Text, IntWritable>(new Text("one"), new IntWritable(1)));
+ oper.input.process(new KeyHashValPair<Text, IntWritable>(new Text("two"), new IntWritable(1)));
+ oper.input.process(new KeyHashValPair<Text, IntWritable>(new Text("two"), new IntWritable(1)));
+ oper.inputCount.process(new KeyHashValPair<Integer, Integer>(1, -1));
+ oper.endWindow();
+ Assert.assertEquals("number emitted tuples", 2, sortSink.collectedTuples.size());
+ for (Object o : sortSink.collectedTuples) {
+ logger.debug(o.toString());
+ }
+ logger.debug("Done testing round\n");
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/mroperator/src/test/java/org/apache/apex/examples/mroperator/WordCountMRApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/mroperator/src/test/java/org/apache/apex/examples/mroperator/WordCountMRApplicationTest.java b/examples/mroperator/src/test/java/org/apache/apex/examples/mroperator/WordCountMRApplicationTest.java
new file mode 100644
index 0000000..fccfa3b
--- /dev/null
+++ b/examples/mroperator/src/test/java/org/apache/apex/examples/mroperator/WordCountMRApplicationTest.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.mroperator;
+
+import java.io.File;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+
+import com.google.common.collect.Maps;
+
+import com.datatorrent.api.LocalMode;
+
+public class WordCountMRApplicationTest
+{
+ private static Logger LOG = LoggerFactory.getLogger(WordCountMRApplicationTest.class);
+ @Rule
+ public MapOperatorTest.TestMeta testMeta = new MapOperatorTest.TestMeta();
+
+ @Test
+ public void testSomeMethod() throws Exception
+ {
+ LocalMode lma = LocalMode.newInstance();
+ Configuration conf = new Configuration(false);
+ conf.set("dt.application.WordCountExample.operator.Mapper.dirName", testMeta.testDir);
+ conf.setInt("dt.application.WordCountExample.operator.Mapper.partitionCount", 1);
+ conf.set("dt.application.WordCountExample.operator.Console.filePath", testMeta.testDir);
+ conf.set("dt.application.WordCountExample.operator.Console.outputFileName", "output.txt");
+ lma.prepareDAG(new NewWordCountApplication(), conf);
+ LocalMode.Controller lc = lma.getController();
+ lc.setHeartbeatMonitoringEnabled(false);
+ lc.run(5000);
+ lc.shutdown();
+ List<String> readLines = FileUtils.readLines(new File(testMeta.testDir + "/output.txt"));
+ Map<String,Integer> readMap = Maps.newHashMap();
+ Iterator<String> itr = readLines.iterator();
+ while (itr.hasNext()) {
+ String[] splits = itr.next().split("=");
+ readMap.put(splits[0],Integer.valueOf(splits[1]));
+ }
+ Map<String,Integer> expectedMap = Maps.newHashMap();
+ expectedMap.put("1",2);
+ expectedMap.put("2",2);
+ expectedMap.put("3",2);
+ Assert.assertEquals("expected reduced data ", expectedMap, readMap);
+ LOG.info("read lines {}", readLines);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/mroperator/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/examples/mroperator/src/test/resources/log4j.properties b/examples/mroperator/src/test/resources/log4j.properties
new file mode 100644
index 0000000..cf0d19e
--- /dev/null
+++ b/examples/mroperator/src/test/resources/log4j.properties
@@ -0,0 +1,43 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+log4j.rootLogger=DEBUG,CONSOLE
+
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+log4j.appender.CONSOLE.threshold=${test.log.console.threshold}
+test.log.console.threshold=DEBUG
+
+log4j.appender.RFA=org.apache.log4j.RollingFileAppender
+log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
+log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+log4j.appender.RFA.File=/tmp/app.log
+
+# to enable, add SYSLOG to rootLogger
+log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender
+log4j.appender.SYSLOG.syslogHost=127.0.0.1
+log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout
+log4j.appender.SYSLOG.layout.conversionPattern=${dt.cid} %-5p [%t] %c{2} %x - %m%n
+log4j.appender.SYSLOG.Facility=LOCAL1
+
+log4j.logger.org=info
+#log4j.logger.org.apache.commons.beanutils=warn
+log4j.logger.com.datatorrent=debug
+log4j.logger.org.apache.apex=debug
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/pi/pom.xml
----------------------------------------------------------------------
diff --git a/examples/pi/pom.xml b/examples/pi/pom.xml
new file mode 100644
index 0000000..6c4935f
--- /dev/null
+++ b/examples/pi/pom.xml
@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>malhar-examples-pi</artifactId>
+ <packaging>jar</packaging>
+
+ <name>Apache Apex Malhar Pi Example</name>
+ <description>Apex example applications that calculate the value of Pi. This is a starting point to understand how Apex works.</description>
+
+ <parent>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-examples</artifactId>
+ <version>3.7.0-SNAPSHOT</version>
+ </parent>
+
+ <dependencies>
+ <dependency>
+ <groupId>it.unimi.dsi</groupId>
+ <artifactId>fastutil</artifactId>
+ <version>6.6.4</version>
+ </dependency>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/pi/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/examples/pi/src/assemble/appPackage.xml b/examples/pi/src/assemble/appPackage.xml
new file mode 100644
index 0000000..4138cf2
--- /dev/null
+++ b/examples/pi/src/assemble/appPackage.xml
@@ -0,0 +1,59 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+ <id>appPackage</id>
+ <formats>
+ <format>jar</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>${basedir}/target/</directory>
+ <outputDirectory>/app</outputDirectory>
+ <includes>
+ <include>${project.artifactId}-${project.version}.jar</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/target/deps</directory>
+ <outputDirectory>/lib</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/site/conf</directory>
+ <outputDirectory>/conf</outputDirectory>
+ <includes>
+ <include>*.xml</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/META-INF</directory>
+ <outputDirectory>/META-INF</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/app</directory>
+ <outputDirectory>/app</outputDirectory>
+ </fileSet>
+ </fileSets>
+
+</assembly>
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/pi/src/main/java/org/apache/apex/examples/pi/Application.java
----------------------------------------------------------------------
diff --git a/examples/pi/src/main/java/org/apache/apex/examples/pi/Application.java b/examples/pi/src/main/java/org/apache/apex/examples/pi/Application.java
new file mode 100644
index 0000000..45f2b37
--- /dev/null
+++ b/examples/pi/src/main/java/org/apache/apex/examples/pi/Application.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.pi;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.DAG.Locality;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.lib.io.ConsoleOutputOperator;
+import com.datatorrent.lib.testbench.RandomEventGenerator;
+
+/**
+ * Monte Carlo PI estimation Example : <br>
+ * This application computes value of PI using Monte Carlo pi estimation
+ * formula.
+ * <p>
+ * Running Java Test or Main app in IDE:
+ *
+ * <pre>
+ * LocalMode.runApp(new Application(), 600000); // 10 min run
+ * </pre>
+ *
+ * Run Success : <br>
+ * For successful deployment and run, user should see something like the
+ * following output on the console (since the input sequence of random numbers
+ * can vary from one run to the next, there will be some variation in the
+ * output values):
+ *
+ * <pre>
+ * 3.1430480549199085
+ * 3.1423454157782515
+ * 3.1431377245508982
+ * 3.142078799249531
+ * 2013-06-18 10:43:18,335 [main] INFO stram.StramLocalCluster run - Application finished.
+ * </pre>
+ *
+ * Application DAG : <br>
+ * <img src="doc-files/Application.gif" width=600px > <br>
+ * <br>
+ *
+ * Streaming Window Size : 1000 ms(1 Sec) <br>
+ * Operator Details : <br>
+ * <ul>
+ * <li><b>The rand Operator : </b> This operator generates random integer
+ * between 0-30k. <br>
+ * Class : {@link com.datatorrent.lib.testbench.RandomEventGenerator}<br>
+ * StateFull : No</li>
+ * <li><b>The calc operator : </b> This operator computes value of pi using
+ * monte carlo estimation. <br>
+ * Class : PiCalculateOperator <br>
+ * StateFull : No</li>
+ * <li><b>The operator Console: </b> This operator just outputs the input tuples
+ * to the console (or stdout). You can use other output adapters if needed.<br>
+ * </li>
+ * </ul>
+ *
+ * @since 0.3.2
+ */
+@ApplicationAnnotation(name = "PiExample")
+public class Application implements StreamingApplication
+{
+ private final Locality locality = null;
+
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ RandomEventGenerator rand = dag.addOperator("rand", new RandomEventGenerator());
+ PiCalculateOperator calc = dag.addOperator("picalc", new PiCalculateOperator());
+ ConsoleOutputOperator console = dag.addOperator("console", new ConsoleOutputOperator());
+ dag.addStream("rand_calc", rand.integer_data, calc.input).setLocality(locality);
+ dag.addStream("rand_console",calc.output, console.input).setLocality(locality);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/pi/src/main/java/org/apache/apex/examples/pi/ApplicationAppData.java
----------------------------------------------------------------------
diff --git a/examples/pi/src/main/java/org/apache/apex/examples/pi/ApplicationAppData.java b/examples/pi/src/main/java/org/apache/apex/examples/pi/ApplicationAppData.java
new file mode 100644
index 0000000..fbd196a
--- /dev/null
+++ b/examples/pi/src/main/java/org/apache/apex/examples/pi/ApplicationAppData.java
@@ -0,0 +1,135 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.pi;
+
+import java.net.URI;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.DAG.Locality;
+import com.datatorrent.api.Operator;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.lib.appdata.schemas.SchemaUtils;
+import com.datatorrent.lib.appdata.snapshot.AppDataSnapshotServerMap;
+import com.datatorrent.lib.io.ConsoleOutputOperator;
+import com.datatorrent.lib.io.PubSubWebSocketAppDataQuery;
+import com.datatorrent.lib.io.PubSubWebSocketAppDataResult;
+import com.datatorrent.lib.testbench.RandomEventGenerator;
+
+/**
+ * Monte Carlo PI estimation example : <br>
+ * This application computes value of PI using Monte Carlo pi estimation
+ * formula.
+ * <p>
+ * Very similar to PiExample but data is also written to an App Data operator for visualization.
+ * <p>
+ * Running Java Test or Main app in IDE:
+ *
+ * <pre>
+ * LocalMode.runApp(new Application(), 600000); // 10 min run
+ * </pre>
+ *
+ * Run Success : <br>
+ * For successful deployment and run, user should see something like the
+ * following output on the console (since the input sequence of random numbers
+ * can vary from one run to the next, there will be some variation in the
+ * output values):
+ *
+ * <pre>
+ * 3.1430480549199085
+ * 3.1423454157782515
+ * 3.1431377245508982
+ * 3.142078799249531
+ * 2013-06-18 10:43:18,335 [main] INFO stram.StramLocalCluster run - Application finished.
+ * </pre>
+ *
+ * Application DAG : <br>
+ * <img src="doc-files/Application.gif" width=600px > <br>
+ * <br>
+ *
+ * Streaming Window Size : 1000 ms(1 Sec) <br>
+ * Operator Details : <br>
+ * <ul>
+ * <li><b>The rand Operator : </b> This operator generates random integer
+ * between 0-30k. <br>
+ * Class : {@link com.datatorrent.lib.testbench.RandomEventGenerator}<br>
+ * StateFull : No</li>
+ * <li><b>The calc operator : </b> This operator computes value of pi using
+ * monte carlo estimation. <br>
+ * Class : PiCalculateOperator <br>
+ * StateFull : No</li>
+ * <li><b>The operator Console: </b> This operator just outputs the input tuples
+ * to the console (or stdout). You can use other output adapters if needed.<br>
+ * </li>
+ * </ul>
+ *
+ * @since 0.3.2
+ */
+@ApplicationAnnotation(name = "PiExampleAppData")
+public class ApplicationAppData implements StreamingApplication
+{
+ public static final String SNAPSHOT_SCHEMA = "PiExampleDataSchema.json";
+
+ private final Locality locality = null;
+
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ RandomEventGenerator rand = dag.addOperator("rand", new RandomEventGenerator());
+ PiCalculateOperator calc = dag.addOperator("picalc", new PiCalculateOperator());
+
+
+ dag.addStream("rand_calc", rand.integer_data, calc.input).setLocality(locality);
+
+ String gatewayAddress = dag.getValue(DAG.GATEWAY_CONNECT_ADDRESS);
+
+ if (StringUtils.isEmpty(gatewayAddress)) {
+ throw new RuntimeException("Error: No GATEWAY_CONNECT_ADDRESS");
+ }
+
+ URI uri = URI.create("ws://" + gatewayAddress + "/pubsub");
+
+ AppDataSnapshotServerMap snapshotServer = dag.addOperator("SnapshotServer", new AppDataSnapshotServerMap());
+
+ String snapshotServerJSON = SchemaUtils.jarResourceFileToString(SNAPSHOT_SCHEMA);
+
+ snapshotServer.setSnapshotSchemaJSON(snapshotServerJSON);
+
+ PubSubWebSocketAppDataQuery wsQuery = new PubSubWebSocketAppDataQuery();
+ wsQuery.enableEmbeddedMode();
+ snapshotServer.setEmbeddableQueryInfoProvider(wsQuery);
+
+ PubSubWebSocketAppDataResult wsResult = dag.addOperator("QueryResult", new PubSubWebSocketAppDataResult());
+
+ wsQuery.setUri(uri);
+ wsResult.setUri(uri);
+ Operator.InputPort<String> queryResultPort = wsResult.input;
+
+ NamedValueList<Object> adaptor = dag.addOperator("adaptor", new NamedValueList<Object>());
+ ConsoleOutputOperator console = dag.addOperator("console", new ConsoleOutputOperator());
+
+ dag.addStream("PiValues", calc.output, adaptor.inPort, console.input).setLocality(locality);;
+ dag.addStream("NamedPiValues", adaptor.outPort, snapshotServer.input);
+ dag.addStream("Result", snapshotServer.queryResult, queryResultPort);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/pi/src/main/java/org/apache/apex/examples/pi/ApplicationWithScript.java
----------------------------------------------------------------------
diff --git a/examples/pi/src/main/java/org/apache/apex/examples/pi/ApplicationWithScript.java b/examples/pi/src/main/java/org/apache/apex/examples/pi/ApplicationWithScript.java
new file mode 100644
index 0000000..8a9cc50
--- /dev/null
+++ b/examples/pi/src/main/java/org/apache/apex/examples/pi/ApplicationWithScript.java
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.pi;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.lib.io.ConsoleOutputOperator;
+import com.datatorrent.lib.script.JavaScriptOperator;
+import com.datatorrent.lib.stream.RoundRobinHashMap;
+import com.datatorrent.lib.testbench.RandomEventGenerator;
+
+/**
+ * Monte Carlo PI estimation example : <br>
+ * This application computes value of PI using Monte Carlo pi estimation
+ * formula. This example inputs formula using java script operator.
+ *
+ * Running Java Test or Main app in IDE:
+ *
+ * <pre>
+ * LocalMode.runApp(new Application(), 600000); // 10 min run
+ * </pre>
+ *
+ * Run Success : <br>
+ * For successful deployment and run, user should see following output on
+ * console:
+ * <pre>
+ * 2013-06-25 11:44:25,842 [container-2] DEBUG stram.StramChildAgent updateOperatorStatus - container-2 pendingDeploy []
+ * 2013-06-25 11:44:25,929 [ServerHelper-1-1] DEBUG netlet.AbstractClient send - allocating new sendBuffer4Offers of size 16384 for Server.Subscriber{type=rrhm_calc/3.inBindings, mask=0, partitions=null}
+ * 3.16
+ * 3.15
+ * 3.1616
+ * 3.148
+ * 3.1393846153846154
+ * </pre>
+ *
+ * * Application DAG : <br>
+ * <img src="doc-files/ApplicationScript.gif" width=600px > <br>
+ * <br>
+ *
+ * Streaming Window Size : 1000 ms(1 Sec) <br>
+ * Operator Details : <br>
+ * <ul>
+ * <li><b>The rand Operator : </b> This operator generates random integer
+ * between 0-30k. <br>
+ * Class : {@link com.datatorrent.lib.testbench.RandomEventGenerator} <br>
+ * StateFull : No</li>
+ * <li><b>The rrhm Operator : </b> This operator takes input from random generator
+ * creates tuples of (x,y) in round robin fashion. <br>
+ * Class : {@link com.datatorrent.lib.stream.RandomEventGenerator} <br>
+ * StateFull : Yes, tuple is emitted after (x, y) values have been aggregated.</li>
+ * <li><b>The calc operator : </b> This is java script operator implementing <br>
+ * Class : {@link com.datatorrent.lib.math.Script} <br>
+ * StateFull : No</li>
+ * <li><b>The operator Console: </b> This operator just outputs the input tuples
+ * to the console (or stdout). User can use any output adapter. <br>
+ * .</li>
+ * </ul>
+ *
+ * @since 0.3.2
+ */
+@ApplicationAnnotation(name = "PiJavaScriptExample")
+public class ApplicationWithScript implements StreamingApplication
+{
+
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ int maxValue = 30000;
+
+ RandomEventGenerator rand = dag.addOperator("rand", new RandomEventGenerator());
+ rand.setMinvalue(0);
+ rand.setMaxvalue(maxValue);
+
+ RoundRobinHashMap<String,Object> rrhm = dag.addOperator("rrhm", new RoundRobinHashMap<String, Object>());
+ rrhm.setKeys(new String[]{"x", "y"});
+
+ JavaScriptOperator calc = dag.addOperator("picalc", new JavaScriptOperator());
+ calc.setPassThru(false);
+ calc.put("i",0);
+ calc.put("count",0);
+ calc.addSetupScript("function pi() { if (x*x+y*y <= " + maxValue * maxValue + ") { i++; } count++; return i / count * 4; }");
+
+ calc.setInvoke("pi");
+
+ dag.addStream("rand_rrhm", rand.integer_data, rrhm.data);
+ dag.addStream("rrhm_calc", rrhm.map, calc.inBindings);
+
+ ConsoleOutputOperator console = dag.addOperator("console", new ConsoleOutputOperator());
+ dag.addStream("rand_console",calc.result, console.input);
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/pi/src/main/java/org/apache/apex/examples/pi/Calculator.java
----------------------------------------------------------------------
diff --git a/examples/pi/src/main/java/org/apache/apex/examples/pi/Calculator.java b/examples/pi/src/main/java/org/apache/apex/examples/pi/Calculator.java
new file mode 100644
index 0000000..672a931
--- /dev/null
+++ b/examples/pi/src/main/java/org/apache/apex/examples/pi/Calculator.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.pi;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.lib.io.ConsoleOutputOperator;
+import com.datatorrent.lib.math.Division;
+import com.datatorrent.lib.math.LogicalCompareToConstant;
+import com.datatorrent.lib.math.MultiplyByConstant;
+import com.datatorrent.lib.math.RunningAverage;
+import com.datatorrent.lib.math.Sigma;
+import com.datatorrent.lib.math.SquareCalculus;
+import com.datatorrent.lib.stream.AbstractAggregator;
+import com.datatorrent.lib.stream.ArrayListAggregator;
+import com.datatorrent.lib.stream.Counter;
+import com.datatorrent.lib.testbench.RandomEventGenerator;
+
+/**
+ * <p>Calculator class.</p>
+ *
+ * @since 0.3.2
+ */
+@ApplicationAnnotation(name = "PiLibraryExample")
+public class Calculator implements StreamingApplication
+{
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ /* keep generating random values between 0 and 30000 */
+ RandomEventGenerator xyGenerator = dag.addOperator("GenerateX", RandomEventGenerator.class);
+
+ /* calculate square of each of the values it receives */
+ SquareCalculus squareOperator = dag.addOperator("SquareX", SquareCalculus.class);
+
+ /* pair the consecutive values */
+ AbstractAggregator<Integer> pairOperator = dag.addOperator("PairXY", new ArrayListAggregator<Integer>());
+ Sigma<Integer> sumOperator = dag.addOperator("SumXY", new Sigma<Integer>());
+ LogicalCompareToConstant<Integer> comparator = dag.addOperator("AnalyzeLocation", new LogicalCompareToConstant<Integer>());
+ comparator.setConstant(30000 * 30000);
+ Counter inCircle = dag.addOperator("CountInCircle", Counter.class);
+ Counter inSquare = dag.addOperator("CountInSquare", Counter.class);
+ Division division = dag.addOperator("Ratio", Division.class);
+ MultiplyByConstant multiplication = dag.addOperator("InstantPI", MultiplyByConstant.class);
+ multiplication.setMultiplier(4);
+ RunningAverage average = dag.addOperator("AveragePI", new RunningAverage());
+ ConsoleOutputOperator oper = dag.addOperator("Console", new ConsoleOutputOperator());
+
+ dag.addStream("x", xyGenerator.integer_data, squareOperator.input);
+ dag.addStream("sqr", squareOperator.integerResult, pairOperator.input);
+ dag.addStream("x2andy2", pairOperator.output, sumOperator.input);
+ dag.addStream("x2plusy2", sumOperator.integerResult, comparator.input, inSquare.input);
+ dag.addStream("inCirclePoints", comparator.greaterThan, inCircle.input);
+ dag.addStream("numerator", inCircle.output, division.numerator);
+ dag.addStream("denominator", inSquare.output, division.denominator);
+ dag.addStream("ratio", division.doubleQuotient, multiplication.input);
+ dag.addStream("instantPi", multiplication.doubleProduct, average.input);
+ dag.addStream("averagePi", average.doubleAverage, oper.input);
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/pi/src/main/java/org/apache/apex/examples/pi/NamedValueList.java
----------------------------------------------------------------------
diff --git a/examples/pi/src/main/java/org/apache/apex/examples/pi/NamedValueList.java b/examples/pi/src/main/java/org/apache/apex/examples/pi/NamedValueList.java
new file mode 100644
index 0000000..aef8a0c
--- /dev/null
+++ b/examples/pi/src/main/java/org/apache/apex/examples/pi/NamedValueList.java
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.pi;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import javax.validation.constraints.NotNull;
+
+import com.datatorrent.api.Context.OperatorContext;
+import com.datatorrent.api.DefaultInputPort;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.common.util.BaseOperator;
+
+/**
+ * <p>An operator which converts a raw value to a named value singleton list.</p>
+ * AppDataSnapshotServerMap.input accepts a List<Map<String,Object>> so we use this operator to
+ * convert individual values to a singleton list of a named value
+ * <p>
+ * @displayNamed Value
+ * @tags count
+ * @since 3.2.0
+ */
+public class NamedValueList<T> extends BaseOperator
+{
+ @NotNull
+ private String valueName;
+
+ private List<Map<String, T>> valueList;
+ private Map<String, T> valueMap;
+
+ public final transient DefaultInputPort<T> inPort = new DefaultInputPort<T>()
+ {
+ @Override
+ public void process(T val)
+ {
+ valueMap.put(valueName, val);
+ outPort.emit(valueList);
+ }
+ };
+
+ public final transient DefaultOutputPort<List<Map<String, T>>> outPort = new DefaultOutputPort<>();
+
+ @Override
+ public void setup(OperatorContext context)
+ {
+ valueMap = new HashMap<>();
+ valueMap.put(valueName, null);
+ valueList = Collections.singletonList(valueMap);
+ }
+
+ @Override
+ public void teardown()
+ {
+ }
+
+ @Override
+ public void beginWindow(long windowId)
+ {
+ }
+
+ @Override
+ public void endWindow()
+ {
+ }
+
+ public String getValueName()
+ {
+ return valueName;
+ }
+
+ public void setValueName(String name)
+ {
+ valueName = name;
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/pi/src/main/java/org/apache/apex/examples/pi/PiCalculateOperator.java
----------------------------------------------------------------------
diff --git a/examples/pi/src/main/java/org/apache/apex/examples/pi/PiCalculateOperator.java b/examples/pi/src/main/java/org/apache/apex/examples/pi/PiCalculateOperator.java
new file mode 100644
index 0000000..b710a96
--- /dev/null
+++ b/examples/pi/src/main/java/org/apache/apex/examples/pi/PiCalculateOperator.java
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.pi;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.datatorrent.api.Context.OperatorContext;
+import com.datatorrent.api.DefaultInputPort;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.common.util.BaseOperator;
+
+/**
+ * This operator implements Monte Carlo estimation of pi. For points randomly distributed points on
+ * square circle. pi ~= Number of points in circle/Total number of points * 4.
+ *
+ * @since 0.3.2
+ */
+public class PiCalculateOperator extends BaseOperator
+{
+ private transient int x = -1;
+ private transient int y = -1;
+ private int base;
+ private long inArea = 0;
+ private long totalArea = 0;
+ public final transient DefaultInputPort<Integer> input = new DefaultInputPort<Integer>()
+ {
+ @Override
+ public void process(Integer tuple)
+ {
+ if (x == -1) {
+ x = tuple;
+ } else {
+ y = tuple;
+ if (x * x + y * y <= base) {
+ inArea++;
+ }
+ totalArea++;
+ x = y = -1;
+ }
+ }
+
+ };
+ public final transient DefaultOutputPort<Double> output = new DefaultOutputPort<Double>();
+
+ @Override
+ public void setup(OperatorContext context)
+ {
+ logger.info("inArea {} totalArea {}", inArea, totalArea);
+ }
+
+ @Override
+ public void beginWindow(long windowId)
+ {
+ }
+
+ @Override
+ public void endWindow()
+ {
+ output.emit((double)inArea / totalArea * 4);
+ }
+
+ public void setBase(int num)
+ {
+ base = num;
+ }
+
+ public int getBase()
+ {
+ return base;
+ }
+
+ private static Logger logger = LoggerFactory.getLogger(PiCalculateOperator.class);
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/pi/src/main/java/org/apache/apex/examples/pi/doc-files/Application.gif
----------------------------------------------------------------------
diff --git a/examples/pi/src/main/java/org/apache/apex/examples/pi/doc-files/Application.gif b/examples/pi/src/main/java/org/apache/apex/examples/pi/doc-files/Application.gif
new file mode 100644
index 0000000..9545c6c
Binary files /dev/null and b/examples/pi/src/main/java/org/apache/apex/examples/pi/doc-files/Application.gif differ
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/pi/src/main/java/org/apache/apex/examples/pi/doc-files/ApplicationScript.gif
----------------------------------------------------------------------
diff --git a/examples/pi/src/main/java/org/apache/apex/examples/pi/doc-files/ApplicationScript.gif b/examples/pi/src/main/java/org/apache/apex/examples/pi/doc-files/ApplicationScript.gif
new file mode 100644
index 0000000..29f9ef8
Binary files /dev/null and b/examples/pi/src/main/java/org/apache/apex/examples/pi/doc-files/ApplicationScript.gif differ
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/pi/src/main/java/org/apache/apex/examples/pi/package-info.java
----------------------------------------------------------------------
diff --git a/examples/pi/src/main/java/org/apache/apex/examples/pi/package-info.java b/examples/pi/src/main/java/org/apache/apex/examples/pi/package-info.java
new file mode 100644
index 0000000..3ebad48
--- /dev/null
+++ b/examples/pi/src/main/java/org/apache/apex/examples/pi/package-info.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/**
+ * Pi calculation demonstration application.
+ */
+package org.apache.apex.examples.pi;
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/pi/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/examples/pi/src/main/resources/META-INF/properties.xml b/examples/pi/src/main/resources/META-INF/properties.xml
new file mode 100644
index 0000000..95df4c7
--- /dev/null
+++ b/examples/pi/src/main/resources/META-INF/properties.xml
@@ -0,0 +1,109 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<configuration>
+ <!-- Memory settings for all examples -->
+ <property>
+ <name>dt.attr.MASTER_MEMORY_MB</name>
+ <value>1024</value>
+ </property>
+ <property>
+ <name>dt.application.*.operator.*.attr.MEMORY_MB</name>
+ <value>256</value>
+ </property>
+ <property>
+ <name>dt.application.*.operator.*.attr.JVM_OPTIONS</name>
+ <value>-Xmx128M</value>
+ </property>
+ <property>
+ <name>dt.application.*.operator.*.port.*.attr.BUFFER_MEMORY_MB</name>
+ <value>128</value>
+ </property>
+
+ <!-- PiExample -->
+ <property>
+ <name>dt.application.PiExample.operator.rand.minvalue</name>
+ <value>0</value>
+ </property>
+ <property>
+ <name>dt.application.PiExample.operator.rand.maxvalue</name>
+ <value>30000</value>
+ </property>
+ <property>
+ <name>dt.application.PiExample.operator.picalc.base</name>
+ <value>900000000</value>
+ </property>
+ <property>
+ <name>dt.application.PiExample.operator.adaptor.valueName</name>
+ <value>piValue</value>
+ </property>
+
+ <!-- PiExampleAppData -->
+ <property>
+ <name>dt.application.PiExampleAppData.operator.rand.minvalue</name>
+ <value>0</value>
+ </property>
+ <property>
+ <name>dt.application.PiExampleAppData.operator.rand.maxvalue</name>
+ <value>30000</value>
+ </property>
+ <property>
+ <name>dt.application.PiExampleAppData.operator.picalc.base</name>
+ <value>900000000</value>
+ </property>
+ <property>
+ <name>dt.application.PiExampleAppData.operator.adaptor.valueName</name>
+ <value>piValue</value>
+ </property>
+ <property>
+ <name>dt.application.PiExampleAppData.operator.Query.topic</name>
+ <value>PiExampleQuery</value>
+ </property>
+ <property>
+ <name>dt.application.PiExampleAppData.operator.QueryResult.topic</name>
+ <value>PiExampleQueryResult</value>
+ </property>
+ <property>
+ <name>dt.application.PiExampleAppData.operator.SnapshotServer.embeddableQueryInfoProvider.topic</name>
+ <value>PiExampleQuery</value>
+ </property>
+
+ <!-- PiLibraryExample -->
+ <property>
+ <name>dt.application.PiLibraryExample.operator.GenerateX.minvalue</name>
+ <value>0</value>
+ </property>
+ <property>
+ <name>dt.application.PiLibraryExample.operator.GenerateX.maxvalue</name>
+ <value>30000</value>
+ </property>
+ <property>
+ <name>dt.application.PiLibraryExample.operator.PairXY.size</name>
+ <value>2</value>
+ </property>
+
+ <!-- PiJavaScriptExample -->
+ <property>
+ <name>dt.application.PiJavaScriptExample.operator.rand.tuplesBlast</name>
+ <value>9</value>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/pi/src/main/resources/PiExampleDataSchema.json
----------------------------------------------------------------------
diff --git a/examples/pi/src/main/resources/PiExampleDataSchema.json b/examples/pi/src/main/resources/PiExampleDataSchema.json
new file mode 100644
index 0000000..47db8eb
--- /dev/null
+++ b/examples/pi/src/main/resources/PiExampleDataSchema.json
@@ -0,0 +1,3 @@
+{
+ "values": [{"name": "piValue", "type": "double"}]
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/pi/src/main/resources/app/PiJsonExample.json
----------------------------------------------------------------------
diff --git a/examples/pi/src/main/resources/app/PiJsonExample.json b/examples/pi/src/main/resources/app/PiJsonExample.json
new file mode 100644
index 0000000..b1e2972
--- /dev/null
+++ b/examples/pi/src/main/resources/app/PiJsonExample.json
@@ -0,0 +1,52 @@
+{
+ "description": "Pi JSON Example that is intended to demonstrate the capability of specifying an app using JSON",
+ "operators": [
+ {
+ "name": "rand",
+ "class": "com.datatorrent.lib.testbench.RandomEventGenerator",
+ "properties": {
+ "minvalue": 0,
+ "maxvalue": 30000
+ }
+ },
+ {
+ "name": "picalc",
+ "class": "com.datatorrent.examples.pi.PiCalculateOperator",
+ "properties": {
+ "base": 900000000
+ }
+ },
+ {
+ "name": "console",
+ "class": "com.datatorrent.lib.io.ConsoleOutputOperator"
+ }
+ ],
+ "streams": [
+ {
+ "name": "rand_calc",
+ "source": {
+ "operatorName": "rand",
+ "portName": "integer_data"
+ },
+ "sinks": [
+ {
+ "operatorName": "picalc",
+ "portName": "input"
+ }
+ ]
+ },
+ {
+ "name": "calc_console",
+ "source": {
+ "operatorName": "picalc",
+ "portName": "output"
+ },
+ "sinks": [
+ {
+ "operatorName": "console",
+ "portName": "input"
+ }
+ ]
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/pi/src/test/java/org/apache/apex/examples/pi/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/pi/src/test/java/org/apache/apex/examples/pi/ApplicationTest.java b/examples/pi/src/test/java/org/apache/apex/examples/pi/ApplicationTest.java
new file mode 100644
index 0000000..d71288f
--- /dev/null
+++ b/examples/pi/src/test/java/org/apache/apex/examples/pi/ApplicationTest.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.pi;
+
+import org.junit.Test;
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.LocalMode;
+
+/**
+ *
+ */
+public class ApplicationTest
+{
+ @Test
+ public void testSomeMethod() throws Exception
+ {
+ LocalMode lma = LocalMode.newInstance();
+ Configuration conf = new Configuration(false);
+ conf.addResource("dt-site-pi.xml");
+ lma.prepareDAG(new Application(), conf);
+ LocalMode.Controller lc = lma.getController();
+ lc.run(10000);
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/pi/src/test/java/org/apache/apex/examples/pi/ApplicationWithScriptTest.java
----------------------------------------------------------------------
diff --git a/examples/pi/src/test/java/org/apache/apex/examples/pi/ApplicationWithScriptTest.java b/examples/pi/src/test/java/org/apache/apex/examples/pi/ApplicationWithScriptTest.java
new file mode 100644
index 0000000..9551c52
--- /dev/null
+++ b/examples/pi/src/test/java/org/apache/apex/examples/pi/ApplicationWithScriptTest.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.pi;
+
+import org.junit.Test;
+
+import com.datatorrent.api.LocalMode;
+
+/**
+ *
+ */
+public class ApplicationWithScriptTest
+{
+ @Test
+ public void testSomeMethod() throws Exception
+ {
+ LocalMode.runApp(new ApplicationWithScript(), 10000);
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/pi/src/test/java/org/apache/apex/examples/pi/CalculatorTest.java
----------------------------------------------------------------------
diff --git a/examples/pi/src/test/java/org/apache/apex/examples/pi/CalculatorTest.java b/examples/pi/src/test/java/org/apache/apex/examples/pi/CalculatorTest.java
new file mode 100644
index 0000000..34ad387
--- /dev/null
+++ b/examples/pi/src/test/java/org/apache/apex/examples/pi/CalculatorTest.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.pi;
+
+import org.junit.Test;
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.LocalMode;
+
+/**
+ *
+ */
+public class CalculatorTest
+{
+ @Test
+ public void testSomeMethod() throws Exception
+ {
+ LocalMode lma = LocalMode.newInstance();
+ Configuration conf = new Configuration(false);
+ conf.addResource("dt-site-pilibrary.xml");
+ lma.prepareDAG(new Calculator(), conf);
+ LocalMode.Controller lc = lma.getController();
+ lc.run(10000);
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/pi/src/test/resources/dt-site-pi.xml
----------------------------------------------------------------------
diff --git a/examples/pi/src/test/resources/dt-site-pi.xml b/examples/pi/src/test/resources/dt-site-pi.xml
new file mode 100644
index 0000000..6032400
--- /dev/null
+++ b/examples/pi/src/test/resources/dt-site-pi.xml
@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<configuration>
+ <property>
+ <name>dt.application.PiExample.class</name>
+ <value>org.apache.apex.examples.pi.Application</value>
+ <description>An alias for the application</description>
+</property>
+<property>
+ <name>dt.application.PiExample.operator.rand.minvalue</name>
+ <value>0</value>
+</property>
+<property>
+ <name>dt.application.PiExample.operator.rand.maxvalue</name>
+ <value>30000</value>
+</property>
+<property>
+ <name>dt.application.PiExample.operator.picalc.base</name>
+ <value>900000000</value>
+</property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/pi/src/test/resources/dt-site-pilibrary.xml
----------------------------------------------------------------------
diff --git a/examples/pi/src/test/resources/dt-site-pilibrary.xml b/examples/pi/src/test/resources/dt-site-pilibrary.xml
new file mode 100644
index 0000000..8f1ae8b
--- /dev/null
+++ b/examples/pi/src/test/resources/dt-site-pilibrary.xml
@@ -0,0 +1,45 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<configuration>
+ <property>
+ <name>dt.application.PiLibraryExample.class</name>
+ <value>org.apache.apex.examples.pi.Calculator</value>
+ <description>An alias for the application</description>
+</property>
+<property>
+ <name>dt.application.PiLibraryExample.operator.GenerateX.minvalue</name>
+ <value>0</value>
+</property>
+<property>
+ <name>dt.application.PiLibraryExample.operator.GenerateX.maxvalue</name>
+ <value>30000</value>
+</property>
+<property>
+ <name>dt.application.PiLibraryExample.operator.PairXY.size</name>
+ <value>2</value>
+</property>
+<!--
+<property>
+ <name>dt.application.PiLibraryExample.operator.AnalyzeLocation.constant</name>
+ <value>900000000</value>
+</property> -->
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/pi/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/examples/pi/src/test/resources/log4j.properties b/examples/pi/src/test/resources/log4j.properties
new file mode 100644
index 0000000..cf0d19e
--- /dev/null
+++ b/examples/pi/src/test/resources/log4j.properties
@@ -0,0 +1,43 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+log4j.rootLogger=DEBUG,CONSOLE
+
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+log4j.appender.CONSOLE.threshold=${test.log.console.threshold}
+test.log.console.threshold=DEBUG
+
+log4j.appender.RFA=org.apache.log4j.RollingFileAppender
+log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
+log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+log4j.appender.RFA.File=/tmp/app.log
+
+# to enable, add SYSLOG to rootLogger
+log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender
+log4j.appender.SYSLOG.syslogHost=127.0.0.1
+log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout
+log4j.appender.SYSLOG.layout.conversionPattern=${dt.cid} %-5p [%t] %c{2} %x - %m%n
+log4j.appender.SYSLOG.Facility=LOCAL1
+
+log4j.logger.org=info
+#log4j.logger.org.apache.commons.beanutils=warn
+log4j.logger.com.datatorrent=debug
+log4j.logger.org.apache.apex=debug
[21/30] apex-malhar git commit: Renamed demos to examples. Packages
and artifactid names are changed as suggested.
Posted by th...@apache.org.
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/mrmonitor/src/main/java/com/datatorrent/demos/mrmonitor/MRStatusObject.java
----------------------------------------------------------------------
diff --git a/demos/mrmonitor/src/main/java/com/datatorrent/demos/mrmonitor/MRStatusObject.java b/demos/mrmonitor/src/main/java/com/datatorrent/demos/mrmonitor/MRStatusObject.java
deleted file mode 100644
index 481f3dc..0000000
--- a/demos/mrmonitor/src/main/java/com/datatorrent/demos/mrmonitor/MRStatusObject.java
+++ /dev/null
@@ -1,501 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.mrmonitor;
-
-import java.util.LinkedList;
-import java.util.Map;
-import java.util.Queue;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-
-import org.codehaus.jettison.json.JSONArray;
-import org.codehaus.jettison.json.JSONException;
-import org.codehaus.jettison.json.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * <p>
- * MRStatusObject class.
- * </p>
- *
- * @since 0.3.4
- */
-public class MRStatusObject
-{
- private String command;
- /**
- * This stores the Resource Manager/ Task Manager's host information
- */
- private String uri;
- /**
- * This field stores the job id
- */
- private String jobId;
- /**
- * This field stores the api version of the rest apis
- */
- private String apiVersion;
- /**
- * This field stores the hadoop version 1 for 1.x and 2 for 2.x
- */
- private int hadoopVersion;
- /**
- * This field stores the app id for the hadoop 2.x
- */
- private String appId;
- /**
- * This field stores the RM port information for hadoop 2.x / Task Manager server port for hadoop 1.X from where we
- * can get the job information
- */
- private int rmPort;
- /**
- * This field stores the history server information for hadoop 2.x from where we can get the job information
- */
- private int historyServerPort;
- /**
- * This field stores the job information as json object
- */
- private JSONObject jsonObject;
- /**
- * This field tells if the object has been modified
- */
- private boolean modified;
- /**
- * This stores the mapping of map task ids to the TaskObject
- */
- private Map<String, TaskObject> mapJsonObject;
- /**
- * This stores the mapping of reduce task ids to the TaskObject
- */
- private Map<String, TaskObject> reduceJsonObject;
- /**
- * This holds the information about the various metrics like MAP_OUTPUT_RECORDS etc for this job
- */
- private TaskObject metricObject;
-
- /**
- * This holds the number of windows occurred when the new data was retrieved for this job
- */
- private int retrials;
-
- /**
- * The scheduler is used to store the job status every 1 minute
- */
- private transient ScheduledExecutorService statusScheduler;
-
- /**
- * This stores the progress of the map tasks
- */
- Queue<String> mapStatusHistory;
-
- /**
- * This stores the progress of the reduce tasks
- */
- Queue<String> reduceStatusHistory;
-
- /**
- * This stores the history of the physical memory usage
- */
- Queue<String> physicalMemoryStatusHistory;
-
- /**
- * This stores the history of the virtual memory usage
- */
- Queue<String> virtualMemoryStatusHistory;
-
- /**
- * This stores the history of the cpu
- */
- Queue<String> cpuStatusHistory;
-
- /**
- * The number of minutes for which the status history of map and reduce tasks is stored
- */
- private int statusHistoryCount = 60;
-
- /**
- * This field notifies if history status queues have changed over time
- */
- private boolean changedHistoryStatus;
-
- public MRStatusObject()
- {
- retrials = 0;
- modified = true;
- mapJsonObject = new ConcurrentHashMap<String, TaskObject>();
- reduceJsonObject = new ConcurrentHashMap<String, TaskObject>();
- mapStatusHistory = new LinkedList<String>();
- reduceStatusHistory = new LinkedList<String>();
- physicalMemoryStatusHistory = new LinkedList<String>();
- virtualMemoryStatusHistory = new LinkedList<String>();
- cpuStatusHistory = new LinkedList<String>();
- statusScheduler = Executors.newScheduledThreadPool(1);
- statusScheduler.scheduleAtFixedRate(new Runnable()
- {
- @Override
- public void run()
- {
- if (jsonObject != null) {
- changedHistoryStatus = true;
- if (mapStatusHistory.size() > statusHistoryCount) {
- mapStatusHistory.poll();
- reduceStatusHistory.poll();
- physicalMemoryStatusHistory.poll();
- virtualMemoryStatusHistory.poll();
- cpuStatusHistory.poll();
- }
- if (hadoopVersion == 2) {
- try {
- mapStatusHistory.add(jsonObject.getJSONObject("job").getString("mapProgress"));
- reduceStatusHistory.add(jsonObject.getJSONObject("job").getString("reduceProgress"));
- if (metricObject.getJson() != null) {
- JSONArray arr = metricObject.getJson().getJSONObject("jobCounters").getJSONArray("counterGroup");
- int length = arr.length();
- for (int i = 0; i < length; i++) {
- if (arr.getJSONObject(i).get("counterGroupName").equals("org.apache.hadoop.mapreduce.TaskCounter")) {
- JSONArray counters = arr.getJSONObject(i).getJSONArray("counter");
- for (int j = 0; j < counters.length(); j++) {
- JSONObject counterObj = counters.getJSONObject(j);
- if (counterObj.get("name").equals("PHYSICAL_MEMORY_BYTES")) {
- physicalMemoryStatusHistory.add(counterObj.getString("totalCounterValue"));
- } else if (counterObj.get("name").equals("VIRTUAL_MEMORY_BYTES")) {
- virtualMemoryStatusHistory.add(counterObj.getString("totalCounterValue"));
- } else if (counterObj.get("name").equals("CPU_MILLISECONDS")) {
- cpuStatusHistory.add(counterObj.getString("totalCounterValue"));
- }
- }
- break;
- }
- }
- }
- } catch (JSONException e) {
- logger.error("error setting status history {}", e.getMessage());
- }
- } else {
- try {
- mapStatusHistory.add(jsonObject.getJSONObject("mapTaskSummary").getString("progressPercentage"));
- reduceStatusHistory.add(jsonObject.getJSONObject("reduceTaskSummary").getString("progressPercentage"));
- JSONArray arr = jsonObject.getJSONArray("jobCounterGroupsInfo");
- int length = arr.length();
- for (int i = 0; i < length; i++) {
- if (arr.getJSONObject(i).get("groupName").equals("Map-Reduce Framework")) {
- JSONArray counters = arr.getJSONObject(i).getJSONArray("jobCountersInfo");
- for (int j = 0; j < counters.length(); j++) {
- JSONObject counterObj = counters.getJSONObject(j);
- if (counterObj.get("name").equals("Physical memory (bytes) snapshot")) {
- physicalMemoryStatusHistory.add(counterObj.getString("totalValue"));
- } else if (counterObj.get("name").equals("Virtual memory (bytes) snapshot")) {
- virtualMemoryStatusHistory.add(counterObj.getString("totalValue"));
- } else if (counterObj.get("name").equals("CPU time spent (ms)")) {
- cpuStatusHistory.add(counterObj.getString("totalValue"));
- }
- }
- break;
- }
- }
- } catch (JSONException e) {
- logger.error("error setting status history {}", e.getMessage());
- }
- }
- }
- }
- }, 0, 1, TimeUnit.MINUTES);
- }
-
- public Map<String, TaskObject> getMapJsonObject()
- {
- return mapJsonObject;
- }
-
- public void setMapJsonObject(Map<String, TaskObject> mapJsonObject)
- {
- this.mapJsonObject = mapJsonObject;
- }
-
- public Map<String, TaskObject> getReduceJsonObject()
- {
- return reduceJsonObject;
- }
-
- public void setReduceJsonObject(Map<String, TaskObject> reduceJsonObject)
- {
- this.reduceJsonObject = reduceJsonObject;
- }
-
- public String getUri()
- {
- return uri;
- }
-
- public void setUri(String uri)
- {
- this.uri = uri;
- }
-
- public String getJobId()
- {
- return jobId;
- }
-
- public void setJobId(String jobId)
- {
- this.jobId = jobId;
- }
-
- public String getApiVersion()
- {
- return apiVersion;
- }
-
- public void setApiVersion(String apiVersion)
- {
- this.apiVersion = apiVersion;
- }
-
- public int getHadoopVersion()
- {
- return hadoopVersion;
- }
-
- public void setHadoopVersion(int hadoopVersion)
- {
- this.hadoopVersion = hadoopVersion;
- }
-
- public String getAppId()
- {
- return appId;
- }
-
- public void setAppId(String appId)
- {
- this.appId = appId;
- }
-
- public int getRmPort()
- {
- return rmPort;
- }
-
- public void setRmPort(int rmPort)
- {
- this.rmPort = rmPort;
- }
-
- public int getHistoryServerPort()
- {
- return historyServerPort;
- }
-
- public void setHistoryServerPort(int historyServerPort)
- {
- this.historyServerPort = historyServerPort;
- }
-
- public JSONObject getJsonObject()
- {
- return jsonObject;
- }
-
- public void setJsonObject(JSONObject jsonObject)
- {
- this.jsonObject = jsonObject;
- }
-
- public boolean isChangedHistoryStatus()
- {
- return changedHistoryStatus;
- }
-
- public void setChangedHistoryStatus(boolean changedHistoryStatus)
- {
- this.changedHistoryStatus = changedHistoryStatus;
- }
-
- @Override
- public boolean equals(Object that)
- {
- if (this == that) {
- return true;
- }
- if (!(that instanceof MRStatusObject)) {
- return false;
- }
- if (this.hashCode() == that.hashCode()) {
- return true;
- }
- return false;
- }
-
- @Override
- public int hashCode()
- {
- return (uri + jobId + apiVersion + String.valueOf(hadoopVersion)).hashCode();
-
- }
-
- public String getCommand()
- {
- return command;
- }
-
- public void setCommand(String command)
- {
- this.command = command;
- }
-
- public boolean isModified()
- {
- return modified;
- }
-
- public void setModified(boolean modified)
- {
- this.modified = modified;
- }
-
- public int getRetrials()
- {
- return retrials;
- }
-
- public void setRetrials(int retrials)
- {
- this.retrials = retrials;
- }
-
- public TaskObject getMetricObject()
- {
- return metricObject;
- }
-
- public void setMetricObject(TaskObject metricObject)
- {
- this.metricObject = metricObject;
- }
-
- public int getStatusHistoryCount()
- {
- return statusHistoryCount;
- }
-
- public void setStatusHistoryCount(int statusHistoryCount)
- {
- this.statusHistoryCount = statusHistoryCount;
- }
-
- public Queue<String> getMapStatusHistory()
- {
- return mapStatusHistory;
- }
-
- public Queue<String> getReduceStatusHistory()
- {
- return reduceStatusHistory;
- }
-
- public Queue<String> getPhysicalMemeoryStatusHistory()
- {
- return physicalMemoryStatusHistory;
- }
-
- public Queue<String> getVirtualMemoryStatusHistory()
- {
- return virtualMemoryStatusHistory;
- }
-
- public Queue<String> getCpuStatusHistory()
- {
- return cpuStatusHistory;
- }
-
- public static class TaskObject
- {
- /**
- * This field stores the task information as json
- */
- private JSONObject json;
- /**
- * This field tells if the object was modified
- */
- private boolean modified;
-
- public TaskObject(JSONObject json)
- {
- modified = true;
- this.json = json;
- }
-
- /**
- * This returns the task information as json
- *
- * @return
- */
- public JSONObject getJson()
- {
- return json;
- }
-
- /**
- * This stores the task information as json
- *
- * @param json
- */
- public void setJson(JSONObject json)
- {
- this.json = json;
- }
-
- /**
- * This returns if the json object has been modified
- *
- * @return
- */
- public boolean isModified()
- {
- return modified;
- }
-
- /**
- * This sets if the json object is modified
- *
- * @param modified
- */
- public void setModified(boolean modified)
- {
- this.modified = modified;
- }
-
- /**
- * This returns the string format of the json object
- *
- * @return
- */
- public String getJsonString()
- {
- return json.toString();
- }
- }
-
- private static Logger logger = LoggerFactory.getLogger(MRStatusObject.class);
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/mrmonitor/src/main/java/com/datatorrent/demos/mrmonitor/MRUtil.java
----------------------------------------------------------------------
diff --git a/demos/mrmonitor/src/main/java/com/datatorrent/demos/mrmonitor/MRUtil.java b/demos/mrmonitor/src/main/java/com/datatorrent/demos/mrmonitor/MRUtil.java
deleted file mode 100644
index 0d7f6af..0000000
--- a/demos/mrmonitor/src/main/java/com/datatorrent/demos/mrmonitor/MRUtil.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.mrmonitor;
-
-import java.io.IOException;
-
-import org.codehaus.jettison.json.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.http.client.ClientProtocolException;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.ResponseHandler;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.impl.client.BasicResponseHandler;
-import org.apache.http.impl.client.DefaultHttpClient;
-
-/**
- * <p>
- * Util class.
- * </p>
- *
- * @since 0.3.4
- */
-public class MRUtil
-{
-
- private static final Logger logger = LoggerFactory.getLogger(MRUtil.class);
-
- /**
- * This method returns the response content for a given url
- * @param url
- * @return
- */
- public static String getJsonForURL(String url)
- {
- HttpClient httpclient = new DefaultHttpClient();
- logger.debug(url);
- try {
-
-
- HttpGet httpget = new HttpGet(url);
-
- // Create a response handler
- ResponseHandler<String> responseHandler = new BasicResponseHandler();
- String responseBody;
- try {
- responseBody = httpclient.execute(httpget, responseHandler);
-
- } catch (ClientProtocolException e) {
- logger.debug(e.getMessage());
- return null;
-
- } catch (IOException e) {
- logger.debug(e.getMessage());
- return null;
- } catch (Exception e) {
- logger.debug(e.getMessage());
- return null;
- }
- return responseBody.trim();
- } finally {
- httpclient.getConnectionManager().shutdown();
- }
- }
-
- /**
- * This method returns the JSONObject for a given string
- * @param json
- * @return
- */
- public static JSONObject getJsonObject(String json)
- {
- try {
- JSONObject jsonObj = new JSONObject(json);
- return jsonObj;
- } catch (Exception e) {
- logger.debug("{}", e.getMessage());
- return null;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/mrmonitor/src/main/java/com/datatorrent/demos/mrmonitor/MapToMRObjectOperator.java
----------------------------------------------------------------------
diff --git a/demos/mrmonitor/src/main/java/com/datatorrent/demos/mrmonitor/MapToMRObjectOperator.java b/demos/mrmonitor/src/main/java/com/datatorrent/demos/mrmonitor/MapToMRObjectOperator.java
deleted file mode 100644
index 5075163..0000000
--- a/demos/mrmonitor/src/main/java/com/datatorrent/demos/mrmonitor/MapToMRObjectOperator.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.mrmonitor;
-
-import java.util.Map;
-
-import com.datatorrent.api.Context.OperatorContext;
-import com.datatorrent.api.DefaultInputPort;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.api.Operator;
-
-/**
- * <p>MapToMRObjectOperator class.</p>
- *
- * @since 0.9.0
- */
-public class MapToMRObjectOperator implements Operator
-{
-
- public final transient DefaultInputPort<Map<String, String>> input = new DefaultInputPort<Map<String, String>>()
- {
- @Override
- public void process(Map<String, String> tuple)
- {
- MRStatusObject mrStatusObj = new MRStatusObject();
-
- for (Map.Entry<String, String> e : tuple.entrySet()) {
- if (e.getKey().equals(Constants.QUERY_KEY_COMMAND)) {
- mrStatusObj.setCommand(e.getValue());
- } else if (e.getKey().equals(Constants.QUERY_API_VERSION)) {
- mrStatusObj.setApiVersion(e.getValue());
- } else if (e.getKey().equals(Constants.QUERY_APP_ID)) {
- mrStatusObj.setAppId(e.getValue());
- } else if (e.getKey().equals(Constants.QUERY_HADOOP_VERSION)) {
- mrStatusObj.setHadoopVersion(Integer.parseInt(e.getValue()));
- } else if (e.getKey().equals(Constants.QUERY_HOST_NAME)) {
- mrStatusObj.setUri(e.getValue());
- } else if (e.getKey().equals(Constants.QUERY_HS_PORT)) {
- mrStatusObj.setHistoryServerPort(Integer.parseInt(e.getValue()));
- } else if (e.getKey().equals(Constants.QUERY_JOB_ID)) {
- mrStatusObj.setJobId(e.getValue());
- } else if (e.getKey().equals(Constants.QUERY_RM_PORT)) {
- mrStatusObj.setRmPort(Integer.parseInt(e.getValue()));
- }
- }
- output.emit(mrStatusObj);
-
- }
- };
-
- public final transient DefaultOutputPort<MRStatusObject> output = new DefaultOutputPort<MRStatusObject>();
-
- @Override
- public void setup(OperatorContext context)
- {
- }
-
- @Override
- public void teardown()
- {
- }
-
- @Override
- public void beginWindow(long windowId)
- {
- }
-
- @Override
- public void endWindow()
- {
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/mrmonitor/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/demos/mrmonitor/src/main/resources/META-INF/properties.xml b/demos/mrmonitor/src/main/resources/META-INF/properties.xml
deleted file mode 100644
index 66de05e..0000000
--- a/demos/mrmonitor/src/main/resources/META-INF/properties.xml
+++ /dev/null
@@ -1,63 +0,0 @@
-<?xml version="1.0"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<configuration>
- <property>
- <name>dt.application.MRMonitoringDemo.operator.JobMonitor.attr.PARTITIONER</name>
- <value>com.datatorrent.common.partitioner.StatelessPartitioner:1</value>
- </property>
- <property>
- <name>dt.application.MRMonitoringDemo.operator.JobMonitor.attr.APPLICATION_WINDOW_COUNT</name>
- <value>4</value>
- </property>
- <property>
- <name>dt.application.MRMonitoringDemo.operator.JobMonitor.prop.maxJobs</name>
- <value>25</value>
- </property>
- <property>
- <name>dt.application.MRMonitoringDemo.operator.JobMonitor.prop.maxJobs</name>
- <value>25</value>
- </property>
- <property>
- <name>dt.application.MRMonitoringDemo.operator.Query.prop.topic</name>
- <value>contrib.summit.mrDebugger.mrDebuggerQuery</value>
- </property>
- <property>
- <name>dt.application.MRMonitoringDemo.operator.JobOutput.prop.topic</name>
- <value>contrib.summit.mrDebugger.jobResult</value>
- </property>
- <property>
- <name>dt.application.MRMonitoringDemo.operator.MapJob.prop.topic</name>
- <value>contrib.summit.mrDebugger.mapResult</value>
- </property>
- <property>
- <name>dt.application.MRMonitoringDemo.operator.ReduceJob.prop.topic</name>
- <value>contrib.summit.mrDebugger.reduceResult</value>
- </property>
- <property>
- <name>dt.application.MRMonitoringDemo.operator.JobCounter.prop.topic</name>
- <value>contrib.summit.mrDebugger.counterResult</value>
- </property>
- <property>
- <name>dt.application.MRMonitoringDemo.stream.QueryConversion.locality</name>
- <value>CONTAINER_LOCAL</value>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/mrmonitor/src/main/resources/mrdebugger.html
----------------------------------------------------------------------
diff --git a/demos/mrmonitor/src/main/resources/mrdebugger.html b/demos/mrmonitor/src/main/resources/mrdebugger.html
deleted file mode 100644
index d8e6497..0000000
--- a/demos/mrmonitor/src/main/resources/mrdebugger.html
+++ /dev/null
@@ -1,237 +0,0 @@
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<!doctype html>
-<html>
-<head>
-<title>Mobile Demo</title>
-
-<META HTTP-EQUIV="CACHE-CONTROL" CONTENT="NO-CACHE">
-<meta name="viewport" content="initial-scale=1.0, user-scalable=no">
-<meta charset="utf-8">
-<script src="http://ajax.googleapis.com/ajax/libs/jquery/1.7.1/jquery.min.js"></script>
-<script src="http://ajax.googleapis.com/ajax/libs/jqueryui/1.10.3/jquery-ui.min.js"></script>
-<!--script src="js/vendor/jquery/dist/jquery.js"></script-->
-<style>
- body {
- margin: 0;
- }
-
- .phone-input {
- margin-left: 0.5em;
- margin-right: 0.5em;
- }
-</style>
-
-</head>
-
-
-<body>
-
-
-<script>
-
-
-
-var map;
-var markers = {};
-
-$(function() {
-
- $("#query1AddButton").click(function() {
-
- var app_id = $("input#app_id").val();
- var hostname = $("input#hostname").val();
- var job_id = $("input#job_id").val();
- var hadoop_version= $("input#hadoop_version").val();
- var api_version = $("input#api_version").val();
- var rm_port = $("input#rm_port").val();
- var hs_port = $("input#hs_port").val();
-
- var jsonData = {
- command : 'add',
- hostname:hostname,
- app_id:app_id,
- job_id:job_id,
- hadoop_version:hadoop_version,
- api_version:api_version,
- rm_port:rm_port,
- hs_port:hs_port
- };
-
- sendQuery(jsonData, function() {
- $('#query1SubmitConfirm').html("<div id='message'></div>");
- $('#message').html("<h2>Add submitted to application!</h2>")
- .append("<p>Result will appear on page shortly.</p>");
- });
-
- return false;
- });
-
- $("#query1DeleteButton").click(function() {
-
- var job_id = $("input#job_id").val();
-
- var jsonData = {
- command : 'delete',
- query : job_id
- };
-
- sendQuery(jsonData, function() {
- $('#query1SubmitConfirm').html("<div id='message'></div>");
- $('#message').html("<h2>Add " + phone + " submitted to application!</h2>")
- .append("<p>Result will appear on page shortly.</p>");
- });
-
- return false;
- });
-
- function sendQuery(jsonData, callback) {
- var ws = new WebSocket('ws://'+window.location.host+'/pubsub');
-
- ws.onopen = function () {
- var topic = "contrib.summit.mrDebugger.mrDebuggerQuery";
- var msg = JSON.stringify({ "type" : "publish", "topic" : topic, "data" : jsonData });
- ws.send(msg);
- console.log("published to: " + topic + " data: " + msg);
- ws.close();
- if (callback) callback();
- };
-
- ws.onerror = function (error) {
- console.log('WebSocket Error ' + error);
- };
-
- ws.onmessage = function (e) {
- console.log('Server: ' + e.data);
- };
- ws.onclose = function (e) {
- console.log('close: ' , e);
- };
-
- }
-
- var ws = new WebSocket('ws://'+window.location.host+'/pubsub');
- var topic = "contrib.summit.mrDebugger.jobResult";
-
- ws.onopen = function () {
- var msg = JSON.stringify({ "type":"subscribe", "topic": topic});
- console.log("sending: " + msg);
- ws.send(msg);
- };
-
- ws.onerror = function (error) {
- console.log('WebSocket Error ' + error);
- };
-
- ws.onmessage = function (e){
-
- $('#jobQueryResult').append(e.data+"\n");
- };
-
-
- var mapws = new WebSocket('ws://'+window.location.host+'/pubsub');
- var maptopic = "contrib.summit.mrDebugger.mapResult";
-
- mapws.onopen = function () {
- var msg = JSON.stringify({ "type":"subscribe", "topic": maptopic});
- console.log("sending: " + msg);
- mapws.send(msg);
- };
-
- mapws.onerror = function (error) {
- console.log('WebSocket Error ' + error);
- };
-
- mapws.onmessage = function (e){
-
- $('#jobMapQueryResult').append(e.data+"\n");
- };
-
-
- var reducews = new WebSocket('ws://'+window.location.host+'/pubsub');
- var reducetopic = "contrib.summit.mrDebugger.reduceResult";
-
- reducews.onopen = function () {
- var msg = JSON.stringify({ "type":"subscribe", "topic": reducetopic});
- console.log("sending: " + msg);
- reducews.send(msg);
- };
-
- reducews.onerror = function (error) {
- console.log('WebSocket Error ' + error);
- };
-
- reducews.onmessage = function (e){
-
- $('#jobReduceQueryResult').append(e.data+"\n");
- };
-
- });
-
-
-</script>
-
-
-<div id="query1FormDiv">
- <form name="query1" action="">
- <p>
- <label for="phone" id="app_id_label">Application Id</label>
- <input type="text" name="app_id" id="app_id" size="30" value="" class="phone-input" />
- </p>
- <p>
- <label for="phone" id="job_id_label">Job Id</label>
- <input type="text" name="job_id" id="job_id" size="30" value="" class="phone-input" />
- </p>
- <p>
- <label for="phone" id="hostname_label">Hostname</label>
- <input type="text" name="hostname" id="hostname" size="30" value="" class="phone-input" />
- </p>
- <p>
- <label for="phone" id="rm_port_label">RM port</label>
- <input type="text" name="rm_port" id="rm_port" size="30" value="" class="phone-input" />
- </p>
- <p>
- <label for="phone" id="hs_port_label">History Server port</label>
- <input type="text" name="hs_port" id="hs_port" size="30" value="" class="phone-input" />
- </p>
- <p>
- <label for="phone" id="hadoop_version_label">Hadoop Version</label>
- <input type="text" name="hadoop_version" id="hadoop_version" size="30" value="" class="phone-input" />
- </p>
- <p>
- <label for="phone" id="api_version_label">API Version</label>
- <input type="text" name="api_version" id="api_version" size="30" value="" class="phone-input" />
- </p>
- <p>
- <input type="submit" name="command" class="button" id="query1AddButton" value="Add" />
- <input type="submit" name="command" class="button" id="query1DeleteButton" value="Delete" />
- <input type="submit" name="command" class="button" id="query1ClearButton" value="Clear" />
- </p>
- </form>
- <div id="query1SubmitConfirm"></div>
- <div>Job: <span id="jobQueryResult"></span></div>
- <div>Map Task: <span id="jobMapQueryResult"></span></div>
- <div>Reduce Job: <span id="jobReduceQueryResult"></span></div>
- </div>
-
-</body>
-</html>
-
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/mrmonitor/src/site/conf/my-app-conf1.xml
----------------------------------------------------------------------
diff --git a/demos/mrmonitor/src/site/conf/my-app-conf1.xml b/demos/mrmonitor/src/site/conf/my-app-conf1.xml
deleted file mode 100644
index f35873b..0000000
--- a/demos/mrmonitor/src/site/conf/my-app-conf1.xml
+++ /dev/null
@@ -1,27 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<configuration>
- <property>
- <name>dt.attr.MASTER_MEMORY_MB</name>
- <value>1024</value>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/mrmonitor/src/test/java/com/datatorrent/demos/mrmonitor/MrMonitoringApplicationTest.java
----------------------------------------------------------------------
diff --git a/demos/mrmonitor/src/test/java/com/datatorrent/demos/mrmonitor/MrMonitoringApplicationTest.java b/demos/mrmonitor/src/test/java/com/datatorrent/demos/mrmonitor/MrMonitoringApplicationTest.java
deleted file mode 100644
index ad8de02..0000000
--- a/demos/mrmonitor/src/test/java/com/datatorrent/demos/mrmonitor/MrMonitoringApplicationTest.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.mrmonitor;
-
-import javax.servlet.Servlet;
-
-import org.eclipse.jetty.server.Connector;
-import org.eclipse.jetty.server.Server;
-import org.eclipse.jetty.servlet.ServletContextHandler;
-import org.eclipse.jetty.servlet.ServletHolder;
-import org.junit.Test;
-
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.LocalMode;
-import com.datatorrent.lib.helper.SamplePubSubWebSocketServlet;
-
-/**
- * <p>MapReduceDebuggerApplicationTest class.</p>
- *
- * @since 0.3.4
- */
-
-public class MrMonitoringApplicationTest
-{
-
- @Test
- public void testApplication() throws Exception
- {
- Configuration conf = new Configuration(false);
- conf.addResource("dt-site-monitoring.xml");
- Server server = new Server(0);
- Servlet servlet = new SamplePubSubWebSocketServlet();
- ServletHolder sh = new ServletHolder(servlet);
- ServletContextHandler contextHandler = new ServletContextHandler(server, "/", ServletContextHandler.SESSIONS);
- contextHandler.addServlet(sh, "/pubsub");
- contextHandler.addServlet(sh, "/*");
- server.start();
- Connector[] connector = server.getConnectors();
- conf.set("dt.attr.GATEWAY_CONNECT_ADDRESS", "localhost:" + connector[0].getLocalPort());
-
- MRMonitoringApplication application = new MRMonitoringApplication();
- LocalMode lma = LocalMode.newInstance();
- lma.prepareDAG(application, conf);
- LocalMode.Controller lc = lma.getController();
- lc.run(10000);
- server.stop();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/mrmonitor/src/test/resources/dt-site-monitoring.xml
----------------------------------------------------------------------
diff --git a/demos/mrmonitor/src/test/resources/dt-site-monitoring.xml b/demos/mrmonitor/src/test/resources/dt-site-monitoring.xml
deleted file mode 100644
index d1e12c6..0000000
--- a/demos/mrmonitor/src/test/resources/dt-site-monitoring.xml
+++ /dev/null
@@ -1,63 +0,0 @@
-<?xml version="1.0"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<configuration>
- <property>
- <name>dt.application.MRMonitoringDemo.operator.JobMonitor.attr.INITIAL_PARTITION_COUNT</name>
- <value>1</value>
- </property>
- <property>
- <name>dt.application.MRMonitoringDemo.operator.JobMonitor.attr.APPLICATION_WINDOW_COUNT</name>
- <value>4</value>
- </property>
- <property>
- <name>dt.application.MRMonitoringDemo.operator.JobMonitor.prop.maxJobs</name>
- <value>25</value>
- </property>
- <property>
- <name>dt.application.MRMonitoringDemo.operator.JobMonitor.prop.maxJobs</name>
- <value>25</value>
- </property>
- <property>
- <name>dt.application.MRMonitoringDemo.operator.Query.prop.topic</name>
- <value>contrib.summit.mrDebugger.mrDebuggerQuery</value>
- </property>
- <property>
- <name>dt.application.MRMonitoringDemo.operator.JobOutput.prop.topic</name>
- <value>contrib.summit.mrDebugger.jobResult</value>
- </property>
- <property>
- <name>dt.application.MRMonitoringDemo.operator.MapJob.prop.topic</name>
- <value>contrib.summit.mrDebugger.mapResult</value>
- </property>
- <property>
- <name>dt.application.MRMonitoringDemo.operator.ReduceJob.prop.topic</name>
- <value>contrib.summit.mrDebugger.reduceResult</value>
- </property>
- <property>
- <name>dt.application.MRMonitoringDemo.operator.JobCounter.prop.topic</name>
- <value>contrib.summit.mrDebugger.counterResult</value>
- </property>
- <property>
- <name>dt.application.MRMonitoringDemo.stream.QueryConversion.locality</name>
- <value>CONTAINER_LOCAL</value>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/mrmonitor/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/demos/mrmonitor/src/test/resources/log4j.properties b/demos/mrmonitor/src/test/resources/log4j.properties
deleted file mode 100644
index cf0d19e..0000000
--- a/demos/mrmonitor/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,43 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-log4j.rootLogger=DEBUG,CONSOLE
-
-log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
-log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
-log4j.appender.CONSOLE.threshold=${test.log.console.threshold}
-test.log.console.threshold=DEBUG
-
-log4j.appender.RFA=org.apache.log4j.RollingFileAppender
-log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
-log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
-log4j.appender.RFA.File=/tmp/app.log
-
-# to enable, add SYSLOG to rootLogger
-log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender
-log4j.appender.SYSLOG.syslogHost=127.0.0.1
-log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout
-log4j.appender.SYSLOG.layout.conversionPattern=${dt.cid} %-5p [%t] %c{2} %x - %m%n
-log4j.appender.SYSLOG.Facility=LOCAL1
-
-log4j.logger.org=info
-#log4j.logger.org.apache.commons.beanutils=warn
-log4j.logger.com.datatorrent=debug
-log4j.logger.org.apache.apex=debug
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/mroperator/pom.xml
----------------------------------------------------------------------
diff --git a/demos/mroperator/pom.xml b/demos/mroperator/pom.xml
deleted file mode 100644
index fcc30ff..0000000
--- a/demos/mroperator/pom.xml
+++ /dev/null
@@ -1,56 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>mroperator</artifactId>
- <packaging>jar</packaging>
-
- <name>Apache Apex Malhar MR Operator Demo</name>
- <description></description>
-
- <parent>
- <groupId>org.apache.apex</groupId>
- <artifactId>malhar-demos</artifactId>
- <version>3.7.0-SNAPSHOT</version>
- </parent>
-
- <properties>
- <skipTests>true</skipTests>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
- <version>${hadoop.version}</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>*</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- </dependencies>
-
-</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/mroperator/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/demos/mroperator/src/assemble/appPackage.xml b/demos/mroperator/src/assemble/appPackage.xml
deleted file mode 100644
index 4138cf2..0000000
--- a/demos/mroperator/src/assemble/appPackage.xml
+++ /dev/null
@@ -1,59 +0,0 @@
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
- <id>appPackage</id>
- <formats>
- <format>jar</format>
- </formats>
- <includeBaseDirectory>false</includeBaseDirectory>
- <fileSets>
- <fileSet>
- <directory>${basedir}/target/</directory>
- <outputDirectory>/app</outputDirectory>
- <includes>
- <include>${project.artifactId}-${project.version}.jar</include>
- </includes>
- </fileSet>
- <fileSet>
- <directory>${basedir}/target/deps</directory>
- <outputDirectory>/lib</outputDirectory>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/site/conf</directory>
- <outputDirectory>/conf</outputDirectory>
- <includes>
- <include>*.xml</include>
- </includes>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/main/resources/META-INF</directory>
- <outputDirectory>/META-INF</outputDirectory>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/main/resources/app</directory>
- <outputDirectory>/app</outputDirectory>
- </fileSet>
- </fileSets>
-
-</assembly>
-
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/DateWritable.java
----------------------------------------------------------------------
diff --git a/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/DateWritable.java b/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/DateWritable.java
deleted file mode 100644
index 5dbd83f..0000000
--- a/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/DateWritable.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.mroperator;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-
-import org.apache.hadoop.io.WritableComparable;
-
-/**
- * <p>DateWritable class.</p>
- *
- * @since 0.9.0
- */
-public class DateWritable implements WritableComparable<DateWritable>
-{
- private static final SimpleDateFormat formatter = new SimpleDateFormat( "yyyy-MM-dd' T 'HH:mm:ss.SSS" );
- private Date date;
-
- public Date getDate()
- {
- return date;
- }
-
- public void setDate( Date date )
- {
- this.date = date;
- }
-
- public void readFields( DataInput in ) throws IOException
- {
- date = new Date( in.readLong() );
- }
-
- public void write( DataOutput out ) throws IOException
- {
- out.writeLong( date.getTime() );
- }
-
- @Override
- public boolean equals(Object o)
- {
- return toString().equals(o.toString());
- }
-
- @Override
- public int hashCode()
- {
- return toString().hashCode();
- }
-
- public String toString()
- {
- return formatter.format( date);
- }
-
- public int compareTo( DateWritable other )
- {
- return date.compareTo( other.getDate() );
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/HdfsKeyValOutputOperator.java
----------------------------------------------------------------------
diff --git a/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/HdfsKeyValOutputOperator.java b/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/HdfsKeyValOutputOperator.java
deleted file mode 100644
index c4b9c49..0000000
--- a/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/HdfsKeyValOutputOperator.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.mroperator;
-
-import com.datatorrent.lib.io.fs.AbstractSingleFileOutputOperator;
-import com.datatorrent.lib.util.KeyHashValPair;
-
-/**
- * Adapter for writing KeyHashValPair objects to HDFS
- * <p>
- * Serializes tuples into a HDFS file.<br/>
- * </p>
- *
- * @param <K> Key type
- * @param <V> Value type
- * @since 0.9.4
- */
-public class HdfsKeyValOutputOperator<K, V> extends AbstractSingleFileOutputOperator<KeyHashValPair<K, V>>
-{
- @Override
- public byte[] getBytesForTuple(KeyHashValPair<K,V> t)
- {
- return (t.toString() + "\n").getBytes();
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/InvertedIndexApplication.java
----------------------------------------------------------------------
diff --git a/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/InvertedIndexApplication.java b/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/InvertedIndexApplication.java
deleted file mode 100644
index 076b8ac..0000000
--- a/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/InvertedIndexApplication.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.mroperator;
-
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.TextInputFormat;
-
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-
-/**
- * <p>InvertedIndexApplication class.</p>
- *
- * @since 0.9.0
- */
-@ApplicationAnnotation(name = "InvertedIndexDemo")
-public class InvertedIndexApplication extends MapReduceApplication<LongWritable, Text, Text, Text>
-{
-
- InvertedIndexApplication()
- {
- setMapClass(LineIndexer.LineIndexMapper.class);
- setReduceClass(LineIndexer.LineIndexReducer.class);
-
- setInputFormat(TextInputFormat.class);
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/LineIndexer.java
----------------------------------------------------------------------
diff --git a/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/LineIndexer.java b/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/LineIndexer.java
deleted file mode 100644
index e963954..0000000
--- a/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/LineIndexer.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.mroperator;
-
-import java.io.IOException;
-import java.util.Iterator;
-import java.util.StringTokenizer;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.FileInputFormat;
-import org.apache.hadoop.mapred.FileOutputFormat;
-import org.apache.hadoop.mapred.FileSplit;
-import org.apache.hadoop.mapred.JobClient;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.MapReduceBase;
-import org.apache.hadoop.mapred.Mapper;
-import org.apache.hadoop.mapred.OutputCollector;
-import org.apache.hadoop.mapred.Reducer;
-import org.apache.hadoop.mapred.Reporter;
-
-/**
- * <p>LineIndexer class.</p>
- *
- * @since 0.9.0
- */
-public class LineIndexer
-{
-
- public static class LineIndexMapper extends MapReduceBase
- implements Mapper<LongWritable, Text, Text, Text>
- {
- private static final Text word = new Text();
- private static final Text location = new Text();
-
- public void map(LongWritable key, Text val,
- OutputCollector<Text, Text> output, Reporter reporter) throws IOException
- {
- FileSplit fileSplit = (FileSplit)reporter.getInputSplit();
- String fileName = fileSplit.getPath().getName();
- location.set(fileName);
-
- String line = val.toString();
- StringTokenizer itr = new StringTokenizer(line.toLowerCase());
- while (itr.hasMoreTokens()) {
- word.set(itr.nextToken());
- output.collect(word, location);
- }
- }
- }
-
-
-
- public static class LineIndexReducer extends MapReduceBase
- implements Reducer<Text, Text, Text, Text>
- {
- public void reduce(Text key, Iterator<Text> values,
- OutputCollector<Text, Text> output, Reporter reporter) throws IOException
- {
- boolean first = true;
- StringBuilder toReturn = new StringBuilder();
- while (values.hasNext()) {
- if (!first) {
- toReturn.append(", ");
- }
- first = false;
- toReturn.append(values.next().toString());
- }
-
- output.collect(key, new Text(toReturn.toString()));
- }
- }
-
-
- /**
- * The actual main() method for our program; this is the
- * "driver" for the MapReduce job.
- */
- public static void main(String[] args)
- {
- JobClient client = new JobClient();
- JobConf conf = new JobConf(LineIndexer.class);
-
- conf.setJobName("LineIndexer");
-
- conf.setOutputKeyClass(Text.class);
- conf.setOutputValueClass(Text.class);
-
- FileInputFormat.addInputPath(conf, new Path("input"));
- FileOutputFormat.setOutputPath(conf, new Path("output"));
-
- conf.setMapperClass(LineIndexMapper.class);
- conf.setReducerClass(LineIndexReducer.class);
-
- client.setConf(conf);
-
- try {
- JobClient.runJob(conf);
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/LogCountsPerHour.java
----------------------------------------------------------------------
diff --git a/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/LogCountsPerHour.java b/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/LogCountsPerHour.java
deleted file mode 100644
index 69ee892..0000000
--- a/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/LogCountsPerHour.java
+++ /dev/null
@@ -1,187 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.mroperator;
-
-import java.io.IOException;
-import java.util.Calendar;
-import java.util.Iterator;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.FileInputFormat;
-import org.apache.hadoop.mapred.FileOutputFormat;
-import org.apache.hadoop.mapred.JobClient;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.MapReduceBase;
-import org.apache.hadoop.mapred.Mapper;
-import org.apache.hadoop.mapred.OutputCollector;
-import org.apache.hadoop.mapred.Reducer;
-import org.apache.hadoop.mapred.Reporter;
-import org.apache.hadoop.mapred.TextOutputFormat;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-
-/**
- * <p>LogCountsPerHour class.</p>
- *
- * @since 0.9.0
- */
-public class LogCountsPerHour extends Configured implements Tool
-{
-
- public static class LogMapClass extends MapReduceBase
- implements Mapper<LongWritable, Text, DateWritable, IntWritable>
- {
- private DateWritable date = new DateWritable();
- private static final IntWritable one = new IntWritable(1);
-
- public void map(LongWritable key, Text value, OutputCollector<DateWritable, IntWritable> output, Reporter reporter) throws IOException
- {
- // Get the value as a String; it is of the format:
- // 111.111.111.111 - - [16/Dec/2012:05:32:50 -0500] "GET / HTTP/1.1" 200 14791 "-" "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)"
- String text = value.toString();
-
- // Get the date and time
- int openBracket = text.indexOf('[');
- int closeBracket = text.indexOf(']');
- if (openBracket != -1 && closeBracket != -1) {
- // Read the date
- String dateString = text.substring(text.indexOf('[') + 1, text.indexOf(']'));
-
- // Build a date object from a string of the form: 16/Dec/2012:05:32:50 -0500
- int index = 0;
- int nextIndex = dateString.indexOf('/');
- int day = Integer.parseInt(dateString.substring(index, nextIndex));
-
- index = nextIndex;
- nextIndex = dateString.indexOf('/', index + 1);
- String month = dateString.substring(index + 1, nextIndex);
-
- index = nextIndex;
- nextIndex = dateString.indexOf(':', index);
- int year = Integer.parseInt(dateString.substring(index + 1, nextIndex));
-
- index = nextIndex;
- nextIndex = dateString.indexOf(':', index + 1);
- int hour = Integer.parseInt(dateString.substring(index + 1, nextIndex));
-
- // Build a calendar object for this date
- Calendar calendar = Calendar.getInstance();
- calendar.set(Calendar.DATE, day);
- calendar.set(Calendar.YEAR, year);
- calendar.set(Calendar.HOUR, hour);
- calendar.set(Calendar.MINUTE, 0);
- calendar.set(Calendar.SECOND, 0);
- calendar.set(Calendar.MILLISECOND, 0);
-
- if (month.equalsIgnoreCase("dec")) {
- calendar.set(Calendar.MONTH, Calendar.DECEMBER);
- } else if (month.equalsIgnoreCase("nov")) {
- calendar.set(Calendar.MONTH, Calendar.NOVEMBER);
- } else if (month.equalsIgnoreCase("oct")) {
- calendar.set(Calendar.MONTH, Calendar.OCTOBER);
- } else if (month.equalsIgnoreCase("sep")) {
- calendar.set(Calendar.MONTH, Calendar.SEPTEMBER);
- } else if (month.equalsIgnoreCase("aug")) {
- calendar.set(Calendar.MONTH, Calendar.AUGUST);
- } else if (month.equalsIgnoreCase("jul")) {
- calendar.set(Calendar.MONTH, Calendar.JULY);
- } else if (month.equalsIgnoreCase("jun")) {
- calendar.set(Calendar.MONTH, Calendar.JUNE);
- } else if (month.equalsIgnoreCase("may")) {
- calendar.set(Calendar.MONTH, Calendar.MAY);
- } else if (month.equalsIgnoreCase("apr")) {
- calendar.set(Calendar.MONTH, Calendar.APRIL);
- } else if (month.equalsIgnoreCase("mar")) {
- calendar.set(Calendar.MONTH, Calendar.MARCH);
- } else if (month.equalsIgnoreCase("feb")) {
- calendar.set(Calendar.MONTH, Calendar.FEBRUARY);
- } else if (month.equalsIgnoreCase("jan")) {
- calendar.set(Calendar.MONTH, Calendar.JANUARY);
- }
-
-
- // Output the date as the key and 1 as the value
- date.setDate(calendar.getTime());
- output.collect(date, one);
- }
- }
- }
-
- public static class LogReduce extends MapReduceBase
- implements Reducer<DateWritable, IntWritable, DateWritable, IntWritable>
- {
- public void reduce(DateWritable key, Iterator<IntWritable> values, OutputCollector<DateWritable, IntWritable> output, Reporter reporter) throws IOException
- {
- // Iterate over all of the values (counts of occurrences of this word)
- int count = 0;
- while (values.hasNext()) {
- // Add the value to our count
- count += values.next().get();
- }
-
- // Output the word with its count (wrapped in an IntWritable)
- output.collect(key, new IntWritable(count));
- }
- }
-
-
- public int run(String[] args) throws Exception
- {
- // Create a configuration
- Configuration conf = getConf();
-
- // Create a job from the default configuration that will use the WordCount class
- JobConf job = new JobConf(conf, LogCountsPerHour.class);
-
- // Define our input path as the first command line argument and our output path as the second
- Path in = new Path(args[0]);
- Path out = new Path(args[1]);
-
- // Create File Input/Output formats for these paths (in the job)
- FileInputFormat.setInputPaths(job, in);
- FileOutputFormat.setOutputPath(job, out);
-
- // Configure the job: name, mapper, reducer, and combiner
- job.setJobName("LogAveragePerHour");
- job.setMapperClass(LogMapClass.class);
- job.setReducerClass(LogReduce.class);
- job.setCombinerClass(LogReduce.class);
-
- // Configure the output
- job.setOutputFormat(TextOutputFormat.class);
- job.setOutputKeyClass(DateWritable.class);
- job.setOutputValueClass(IntWritable.class);
-
- // Run the job
- JobClient.runJob(job);
- return 0;
- }
-
- public static void main(String[] args) throws Exception
- {
- // Start the LogCountsPerHour MapReduce application
- int res = ToolRunner.run(new Configuration(), new LogCountsPerHour(), args);
- System.exit(res);
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/LogsCountApplication.java
----------------------------------------------------------------------
diff --git a/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/LogsCountApplication.java b/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/LogsCountApplication.java
deleted file mode 100644
index 2d647ed..0000000
--- a/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/LogsCountApplication.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.mroperator;
-
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.TextInputFormat;
-
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-
-/**
- * <p>LogsCountApplication class.</p>
- *
- * @since 0.9.0
- */
-@ApplicationAnnotation(name = "LogsCountDemo")
-public class LogsCountApplication extends MapReduceApplication<LongWritable, Text, DateWritable, IntWritable>
-{
-
- public void LogsCountApplication()
- {
- setMapClass(LogCountsPerHour.LogMapClass.class);
- // setCombineClass(LogCountsPerHour.LogReduce.class);
- setReduceClass(LogCountsPerHour.LogReduce.class);
- setInputFormat(TextInputFormat.class);
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/MapOperator.java
----------------------------------------------------------------------
diff --git a/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/MapOperator.java b/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/MapOperator.java
deleted file mode 100644
index 509f6ae..0000000
--- a/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/MapOperator.java
+++ /dev/null
@@ -1,414 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.mroperator;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-import javax.validation.constraints.Min;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.serializer.Deserializer;
-import org.apache.hadoop.io.serializer.SerializationFactory;
-import org.apache.hadoop.io.serializer.Serializer;
-import org.apache.hadoop.mapred.Counters;
-import org.apache.hadoop.mapred.FileInputFormat;
-import org.apache.hadoop.mapred.InputFormat;
-import org.apache.hadoop.mapred.InputSplit;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.KeyValueTextInputFormat;
-import org.apache.hadoop.mapred.Mapper;
-import org.apache.hadoop.mapred.OutputCollector;
-import org.apache.hadoop.mapred.RecordReader;
-import org.apache.hadoop.mapred.Reducer;
-import org.apache.hadoop.mapred.Reporter;
-import org.apache.hadoop.mapred.TextInputFormat;
-
-import com.datatorrent.api.Context.OperatorContext;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.api.DefaultPartition;
-import com.datatorrent.api.InputOperator;
-import com.datatorrent.api.Partitioner;
-
-import com.datatorrent.demos.mroperator.ReporterImpl.ReporterType;
-import com.datatorrent.lib.util.KeyHashValPair;
-
-/**
- * <p>
- * MapOperator class.
- * </p>
- *
- * @since 0.9.0
- */
-@SuppressWarnings({ "unchecked"})
-public class MapOperator<K1, V1, K2, V2> implements InputOperator, Partitioner<MapOperator<K1, V1, K2, V2>>
-{
-
- private static final Logger logger = LoggerFactory.getLogger(MapOperator.class);
- private String dirName;
- private boolean emitPartitioningCountOnce = false;
- private boolean emitLastCountOnce = false;
- private int operatorId;
- private Class<? extends InputFormat<K1, V1>> inputFormatClass;
- private transient InputFormat<K1, V1> inputFormat;
- private transient InputSplit inputSplit;
- private Class<? extends InputSplit> inputSplitClass;
- private ByteArrayOutputStream outstream = new ByteArrayOutputStream();
- private transient RecordReader<K1, V1> reader;
- private boolean emittedAll = false;
- public final transient DefaultOutputPort<KeyHashValPair<Integer, Integer>> outputCount = new DefaultOutputPort<KeyHashValPair<Integer, Integer>>();
- public final transient DefaultOutputPort<KeyHashValPair<K2, V2>> output = new DefaultOutputPort<KeyHashValPair<K2, V2>>();
- private transient JobConf jobConf;
- @Min(1)
- private int partitionCount = 1;
-
- public Class<? extends InputSplit> getInputSplitClass()
- {
- return inputSplitClass;
- }
-
- public void setInputSplitClass(Class<? extends InputSplit> inputSplitClass)
- {
- this.inputSplitClass = inputSplitClass;
- }
-
- public Class<? extends InputFormat<K1, V1>> getInputFormatClass()
- {
- return inputFormatClass;
- }
-
- public void setInputFormatClass(Class<? extends InputFormat<K1, V1>> inputFormatClass)
- {
- this.inputFormatClass = inputFormatClass;
- }
-
- public String getDirName()
- {
- return dirName;
- }
-
- public void setDirName(String dirName)
- {
- this.dirName = dirName;
- }
-
- public int getPartitionCount()
- {
- return partitionCount;
- }
-
- public void setPartitionCount(int partitionCount)
- {
- this.partitionCount = partitionCount;
- }
-
- @Override
- public void beginWindow(long windowId)
- {
- if (!emitPartitioningCountOnce) {
- outputCount.emit(new KeyHashValPair<Integer, Integer>(operatorId, 1));
- emitPartitioningCountOnce = true;
- }
- if (reader == null) {
- try {
- reader = inputFormat.getRecordReader(inputSplit, new JobConf(new Configuration()), reporter);
- } catch (IOException e) {
- logger.info("error getting record reader {}", e.getMessage());
- }
- }
- }
-
- @Override
- public void teardown()
- {
-
- }
-
- @Override
- public void setup(OperatorContext context)
- {
- if (context != null) {
- operatorId = context.getId();
- }
- reporter = new ReporterImpl(ReporterType.Mapper, new Counters());
- outputCollector = new OutputCollectorImpl<K2, V2>();
- Configuration conf = new Configuration();
- try {
- inputFormat = inputFormatClass.newInstance();
- SerializationFactory serializationFactory = new SerializationFactory(conf);
- Deserializer keyDesiralizer = serializationFactory.getDeserializer(inputSplitClass);
- keyDesiralizer.open(new ByteArrayInputStream(outstream.toByteArray()));
- inputSplit = (InputSplit)keyDesiralizer.deserialize(null);
- ((ReporterImpl)reporter).setInputSplit(inputSplit);
- reader = inputFormat.getRecordReader(inputSplit, new JobConf(conf), reporter);
- } catch (Exception e) {
- logger.info("failed to initialize inputformat obj {}", inputFormat);
- throw new RuntimeException(e);
- }
- InputStream stream = null;
- if (configFile != null && configFile.length() > 0) {
- stream = ClassLoader.getSystemResourceAsStream("/" + configFile);
- if (stream == null) {
- stream = ClassLoader.getSystemResourceAsStream(configFile);
- }
- }
- if (stream != null) {
- conf.addResource(stream);
- }
- jobConf = new JobConf(conf);
- if (mapClass != null) {
- try {
- mapObject = mapClass.newInstance();
- } catch (Exception e) {
- logger.info("can't instantiate object {}", e.getMessage());
- }
-
- mapObject.configure(jobConf);
- }
- if (combineClass != null) {
- try {
- combineObject = combineClass.newInstance();
- } catch (Exception e) {
- logger.info("can't instantiate object {}", e.getMessage());
- }
- combineObject.configure(jobConf);
- }
- }
-
- @Override
- public void emitTuples()
- {
- if (!emittedAll) {
- try {
- K1 key = reader.createKey();
- V1 val = reader.createValue();
- emittedAll = !reader.next(key, val);
- if (!emittedAll) {
- KeyHashValPair<K1, V1> keyValue = new KeyHashValPair<K1, V1>(key, val);
- mapObject.map(keyValue.getKey(), keyValue.getValue(), outputCollector, reporter);
- if (combineObject == null) {
- List<KeyHashValPair<K2, V2>> list = ((OutputCollectorImpl<K2, V2>)outputCollector).getList();
- for (KeyHashValPair<K2, V2> e : list) {
- output.emit(e);
- }
- list.clear();
- }
- }
- } catch (IOException ex) {
- logger.debug(ex.toString());
- throw new RuntimeException(ex);
- }
- }
- }
-
- @Override
- public void endWindow()
- {
- List<KeyHashValPair<K2, V2>> list = ((OutputCollectorImpl<K2, V2>)outputCollector).getList();
- if (combineObject != null) {
- Map<K2, List<V2>> cacheObject = new HashMap<K2, List<V2>>();
- for (KeyHashValPair<K2, V2> tuple : list) {
- List<V2> cacheList = cacheObject.get(tuple.getKey());
- if (cacheList == null) {
- cacheList = new ArrayList<V2>();
- cacheList.add(tuple.getValue());
- cacheObject.put(tuple.getKey(), cacheList);
- } else {
- cacheList.add(tuple.getValue());
- }
- }
- list.clear();
- OutputCollector<K2, V2> tempOutputCollector = new OutputCollectorImpl<K2, V2>();
- for (Map.Entry<K2, List<V2>> e : cacheObject.entrySet()) {
- try {
- combineObject.reduce(e.getKey(), e.getValue().iterator(), tempOutputCollector, reporter);
- } catch (IOException e1) {
- logger.info(e1.getMessage());
- }
- }
- list = ((OutputCollectorImpl<K2, V2>)tempOutputCollector).getList();
- for (KeyHashValPair<K2, V2> e : list) {
- output.emit(e);
- }
- }
- if (!emitLastCountOnce && emittedAll) {
- outputCount.emit(new KeyHashValPair<Integer, Integer>(operatorId, -1));
- logger.info("emitting end of file {}", new KeyHashValPair<Integer, Integer>(operatorId, -1));
- emitLastCountOnce = true;
- }
- list.clear();
- }
-
- private InputSplit[] getSplits(JobConf conf, int numSplits, String path) throws Exception
- {
- FileInputFormat.setInputPaths(conf, new Path(path));
- if (inputFormat == null) {
- inputFormat = inputFormatClass.newInstance();
- String inputFormatClassName = inputFormatClass.getName();
- if (inputFormatClassName.equals("org.apache.hadoop.mapred.TextInputFormat")) {
- ((TextInputFormat)inputFormat).configure(conf);
- } else if (inputFormatClassName.equals("org.apache.hadoop.mapred.KeyValueTextInputFormat")) {
- ((KeyValueTextInputFormat)inputFormat).configure(conf);
- }
- }
- return inputFormat.getSplits(conf, numSplits);
- // return null;
- }
-
- @Override
- public void partitioned(Map<Integer, Partition<MapOperator<K1, V1, K2, V2>>> partitions)
- {
- }
-
- @SuppressWarnings("rawtypes")
- @Override
- public Collection<Partition<MapOperator<K1, V1, K2, V2>>> definePartitions(Collection<Partition<MapOperator<K1, V1, K2, V2>>> partitions, PartitioningContext context)
- {
- int tempPartitionCount = partitionCount;
-
- Collection c = partitions;
- Collection<Partition<MapOperator<K1, V1, K2, V2>>> operatorPartitions = c;
- Partition<MapOperator<K1, V1, K2, V2>> template;
- Iterator<Partition<MapOperator<K1, V1, K2, V2>>> itr = operatorPartitions.iterator();
- template = itr.next();
- Configuration conf = new Configuration();
- SerializationFactory serializationFactory = new SerializationFactory(conf);
- if (outstream.size() == 0) {
- InputSplit[] splits;
- try {
- splits = getSplits(new JobConf(conf), tempPartitionCount, template.getPartitionedInstance().getDirName());
- } catch (Exception e1) {
- logger.info(" can't get splits {}", e1.getMessage());
- throw new RuntimeException(e1);
- }
- Collection<Partition<MapOperator<K1, V1, K2, V2>>> operList = new ArrayList<Partition<MapOperator<K1, V1, K2, V2>>>();
- itr = operatorPartitions.iterator();
- int size = splits.length;
- Serializer keySerializer = serializationFactory.getSerializer(splits[0].getClass());
- while (size > 0 && itr.hasNext()) {
- Partition<MapOperator<K1, V1, K2, V2>> p = itr.next();
- MapOperator<K1, V1, K2, V2> opr = p.getPartitionedInstance();
- opr.setInputFormatClass(inputFormatClass);
- opr.setMapClass(mapClass);
- opr.setCombineClass(combineClass);
- opr.setConfigFile(configFile);
- try {
- keySerializer.open(opr.getOutstream());
- keySerializer.serialize(splits[size - 1]);
- opr.setInputSplitClass(splits[size - 1].getClass());
- } catch (IOException e) {
- logger.info("error while serializing {}", e.getMessage());
- }
- size--;
- operList.add(p);
- }
- while (size > 0) {
- MapOperator<K1, V1, K2, V2> opr = new MapOperator<K1, V1, K2, V2>();
- opr.setInputFormatClass(inputFormatClass);
- opr.setMapClass(mapClass);
- opr.setCombineClass(combineClass);
- opr.setConfigFile(configFile);
- try {
- keySerializer.open(opr.getOutstream());
- keySerializer.serialize(splits[size - 1]);
- opr.setInputSplitClass(splits[size - 1].getClass());
- } catch (IOException e) {
- logger.info("error while serializing {}", e.getMessage());
- }
- size--;
- operList.add(new DefaultPartition<MapOperator<K1, V1, K2, V2>>(opr));
- }
- try {
- keySerializer.close();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- return operList;
- }
- return null;
- }
-
- public ByteArrayOutputStream getOutstream()
- {
- return outstream;
- }
-
- public void setOutstream(ByteArrayOutputStream outstream)
- {
- this.outstream = outstream;
- }
-
- /**
- * adding map code
- */
-
- private Class<? extends Mapper<K1, V1, K2, V2>> mapClass;
- private Class<? extends Reducer<K2, V2, K2, V2>> combineClass;
-
- private transient Mapper<K1, V1, K2, V2> mapObject;
- private transient Reducer<K2, V2, K2, V2> combineObject;
- private transient Reporter reporter;
-
- private String configFile;
-
- public String getConfigFile()
- {
- return configFile;
- }
-
- public void setConfigFile(String configFile)
- {
- this.configFile = configFile;
- }
-
- private transient OutputCollector<K2, V2> outputCollector;
-
- public Class<? extends Mapper<K1, V1, K2, V2>> getMapClass()
- {
- return mapClass;
- }
-
- public void setMapClass(Class<? extends Mapper<K1, V1, K2, V2>> mapClass)
- {
- this.mapClass = mapClass;
- }
-
- public Class<? extends Reducer<K2, V2, K2, V2>> getCombineClass()
- {
- return combineClass;
- }
-
- public void setCombineClass(Class<? extends Reducer<K2, V2, K2, V2>> combineClass)
- {
- this.combineClass = combineClass;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/MapReduceApplication.java
----------------------------------------------------------------------
diff --git a/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/MapReduceApplication.java b/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/MapReduceApplication.java
deleted file mode 100644
index 9fffc3f..0000000
--- a/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/MapReduceApplication.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.mroperator;
-
-import java.util.StringTokenizer;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapred.InputFormat;
-import org.apache.hadoop.mapred.Mapper;
-import org.apache.hadoop.mapred.Reducer;
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-
-
-/**
- * <p>
- * Abstract MapReduceApplication class.
- * </p>
- *
- * @since 0.9.0
- */
-@ApplicationAnnotation(name = "MapReduceDemo")
-public abstract class MapReduceApplication<K1, V1, K2, V2> implements StreamingApplication
-{
- Class<? extends InputFormat<K1, V1>> inputFormat;
- Class<? extends Mapper<K1, V1, K2, V2>> mapClass;
- Class<? extends Reducer<K2, V2, K2, V2>> reduceClass;
- Class<? extends Reducer<K2, V2, K2, V2>> combineClass;
-
- public Class<? extends Reducer<K2, V2, K2, V2>> getCombineClass()
- {
- return combineClass;
- }
-
- public void setCombineClass(Class<? extends Reducer<K2, V2, K2, V2>> combineClass)
- {
- this.combineClass = combineClass;
- }
-
- public void setInputFormat(Class<? extends InputFormat<K1, V1>> inputFormat)
- {
- this.inputFormat = inputFormat;
- }
-
- public Class<? extends Mapper<K1, V1, K2, V2>> getMapClass()
- {
- return mapClass;
- }
-
- public void setMapClass(Class<? extends Mapper<K1, V1, K2, V2>> mapClass)
- {
- this.mapClass = mapClass;
- }
-
- public Class<? extends Reducer<K2, V2, K2, V2>> getReduceClass()
- {
- return reduceClass;
- }
-
- public void setReduceClass(Class<? extends Reducer<K2, V2, K2, V2>> reduceClass)
- {
- this.reduceClass = reduceClass;
- }
-
-
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- String configurationFilePath = conf.get(this.getClass().getSimpleName() + ".configFile", "");
-
- MapOperator<K1, V1, K2, V2> inputOperator = dag.addOperator("Mapper", new MapOperator<K1, V1, K2, V2>());
- inputOperator.setInputFormatClass(inputFormat);
-
- String configFileName = null;
- if (configurationFilePath != null && !configurationFilePath.isEmpty()) {
- StringTokenizer configFileTokenizer = new StringTokenizer(configurationFilePath, "/");
- configFileName = configFileTokenizer.nextToken();
- while (configFileTokenizer.hasMoreTokens()) {
- configFileName = configFileTokenizer.nextToken();
- }
- }
-
- inputOperator.setMapClass(mapClass);
- inputOperator.setConfigFile(configFileName);
- inputOperator.setCombineClass(combineClass);
-
- ReduceOperator<K2, V2, K2, V2> reduceOpr = dag.addOperator("Reducer", new ReduceOperator<K2, V2, K2, V2>());
- reduceOpr.setReduceClass(reduceClass);
- reduceOpr.setConfigFile(configFileName);
-
- HdfsKeyValOutputOperator<K2, V2> console = dag.addOperator("Console", new HdfsKeyValOutputOperator<K2, V2>());
-
- dag.addStream("Mapped-Output", inputOperator.output, reduceOpr.input);
- dag.addStream("Mapper-Count", inputOperator.outputCount, reduceOpr.inputCount);
- dag.addStream("Reduced-Output", reduceOpr.output, console.input);
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/NewWordCountApplication.java
----------------------------------------------------------------------
diff --git a/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/NewWordCountApplication.java b/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/NewWordCountApplication.java
deleted file mode 100644
index b0ea7d8..0000000
--- a/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/NewWordCountApplication.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.mroperator;
-
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.TextInputFormat;
-
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-
-/**
- * <p>NewWordCountApplication class.</p>
- *
- * @since 0.9.0
- */
-@ApplicationAnnotation(name = "WordCountDemo")
-public class NewWordCountApplication extends MapReduceApplication<LongWritable, Text, Text, IntWritable>
-{
-
- public void NewWordCountApplication()
- {
- setMapClass(WordCount.Map.class);
- setReduceClass(WordCount.Reduce.class);
- setCombineClass(WordCount.Reduce.class);
- setInputFormat(TextInputFormat.class);
- }
-}
[22/30] apex-malhar git commit: Renamed demos to examples. Packages
and artifactid names are changed as suggested.
Posted by th...@apache.org.
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/mobile/pom.xml
----------------------------------------------------------------------
diff --git a/demos/mobile/pom.xml b/demos/mobile/pom.xml
deleted file mode 100644
index 1d2c66c..0000000
--- a/demos/mobile/pom.xml
+++ /dev/null
@@ -1,64 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>mobile-demo</artifactId>
- <packaging>jar</packaging>
-
- <name>Apache Apex Malhar Mobile Demo</name>
- <description></description>
-
- <parent>
- <groupId>org.apache.apex</groupId>
- <artifactId>malhar-demos</artifactId>
- <version>3.7.0-SNAPSHOT</version>
- </parent>
-
- <properties>
- <skipTests>true</skipTests>
- </properties>
-
- <dependencies>
- <!-- add your dependencies here -->
- <dependency>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-servlet</artifactId>
- <version>8.1.10.v20130312</version>
- <scope>test</scope>
- <type>jar</type>
- </dependency>
- <dependency>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-websocket</artifactId>
- <version>8.1.10.v20130312</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- <version>3.1</version>
- <type>jar</type>
- </dependency>
- </dependencies>
-
-</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/mobile/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/demos/mobile/src/assemble/appPackage.xml b/demos/mobile/src/assemble/appPackage.xml
deleted file mode 100644
index 4138cf2..0000000
--- a/demos/mobile/src/assemble/appPackage.xml
+++ /dev/null
@@ -1,59 +0,0 @@
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
- <id>appPackage</id>
- <formats>
- <format>jar</format>
- </formats>
- <includeBaseDirectory>false</includeBaseDirectory>
- <fileSets>
- <fileSet>
- <directory>${basedir}/target/</directory>
- <outputDirectory>/app</outputDirectory>
- <includes>
- <include>${project.artifactId}-${project.version}.jar</include>
- </includes>
- </fileSet>
- <fileSet>
- <directory>${basedir}/target/deps</directory>
- <outputDirectory>/lib</outputDirectory>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/site/conf</directory>
- <outputDirectory>/conf</outputDirectory>
- <includes>
- <include>*.xml</include>
- </includes>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/main/resources/META-INF</directory>
- <outputDirectory>/META-INF</outputDirectory>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/main/resources/app</directory>
- <outputDirectory>/app</outputDirectory>
- </fileSet>
- </fileSets>
-
-</assembly>
-
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/mobile/src/main/java/com/datatorrent/demos/mobile/Application.java
----------------------------------------------------------------------
diff --git a/demos/mobile/src/main/java/com/datatorrent/demos/mobile/Application.java b/demos/mobile/src/main/java/com/datatorrent/demos/mobile/Application.java
deleted file mode 100644
index 30d7281..0000000
--- a/demos/mobile/src/main/java/com/datatorrent/demos/mobile/Application.java
+++ /dev/null
@@ -1,172 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.mobile;
-
-import java.net.URI;
-import java.util.Arrays;
-import java.util.Map;
-import java.util.Random;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.commons.lang.mutable.MutableLong;
-import org.apache.commons.lang3.Range;
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.Context.OperatorContext;
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.StatsListener;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-import com.datatorrent.lib.counters.BasicCounters;
-import com.datatorrent.lib.io.PubSubWebSocketInputOperator;
-import com.datatorrent.lib.io.PubSubWebSocketOutputOperator;
-import com.datatorrent.lib.partitioner.StatelessThroughputBasedPartitioner;
-import com.datatorrent.lib.testbench.RandomEventGenerator;
-
-/**
- * Mobile Demo Application:
- * <p>
- * This demo simulates large number of cell phones in the range of 40K to 200K
- * and tracks a given cell number across cell towers. It also displays the changing locations of the cell number on a google map.
- *
- * This demo demonstrates the scalability feature of the Apex platform.
- * It showcases the ability of the platform to scale up and down as the phone numbers generated increase and decrease respectively.
- * If the tuples processed per second by the pmove operator increase beyond 30,000, more partitions of the pmove operator gets deployed until
- * each of the partition processes around 10000 to 30000 tuples per second.
- * If the tuples processed per second drops below 10,000, the platform merges the operators until the partition count drops down to the original.
- * The load can be varied using the tuplesBlast property.
- * If the tuplesBlast is set to 200, 40K cell phones are generated.
- * If the tuplesBlast is set to 1000, 200K cell phones are generated.
- * The tuplesBlast property can be set using dtcli command: 'set-operator-property pmove tuplesBlast 1000'.
- *
- *
- * The specs are as such<br>
- * Depending on the tuplesBlast property, large number of cell phone numbers are generated.
- * They jump a cell tower frequently. Sometimes
- * within a second sometimes in 10 seconds. The aim is to demonstrate the
- * following abilities<br>
- * <ul>
- * <li>Entering query dynamically: The phone numbers are added to locate its gps
- * in run time.</li>
- * <li>Changing functionality dynamically: The load is changed by making
- * functional changes on the load generator operator (phonegen)(</li>
- * <li>Auto Scale up/Down with load: Operator pmove increases and decreases
- * partitions as per load</li>
- * <li></li>
- * </ul>
- *
- * Refer to demos/docs/MobileDemo.md for more information.
- *
- * <p>
- *
- * Running Java Test or Main app in IDE:
- *
- * <pre>
- * LocalMode.runApp(new Application(), 600000); // 10 min run
- * </pre>
- *
- * Run Success : <br>
- * For successful deployment and run, user should see following output on
- * console: <br>
- *
- * <pre>
- * phoneLocationQueryResult: {phone=5556101, location=(5,9), queryId=q3}
- * phoneLocationQueryResult: {phone=5554995, location=(10,4), queryId=q1}
- * phoneLocationQueryResult: {phone=5556101, location=(5,9), queryId=q3}
- * phoneLocationQueryResult: {phone=5554995, location=(10,4), queryId=q1}
- * phoneLocationQueryResult: {phone=5554995, location=(10,5), queryId=q1}
- * phoneLocationQueryResult: {phone=5556101, location=(5,9), queryId=q3}
- * phoneLocationQueryResult: {phone=5554995, location=(9,5), queryId=q1}
- * phoneLocationQueryResult: {phone=5556101, location=(5,9), queryId=q3}
- * phoneLocationQueryResult: {phone=5556101, location=(5,9), queryId=q3}
- * phoneLocationQueryResult: {phone=5554995, location=(9,5), queryId=q1}
- * phoneLocationQueryResult: {phone=5554995, location=(9,5), queryId=q1}
- * phoneLocationQueryResult: {phone=5556101, location=(5,9), queryId=q3}
- * </pre>
- *
- * * <b>Application DAG : </b><br>
- * <img src="doc-files/mobile.png" width=600px > <br>
- *
- * @since 0.3.2
- */
-@ApplicationAnnotation(name = "MobileDemo")
-public class Application implements StreamingApplication
-{
- public static final String PHONE_RANGE_PROP = "dt.application.MobileDemo.phoneRange";
- public static final String TOTAL_SEED_NOS = "dt.application.MobileDemo.totalSeedNumbers";
- public static final String COOL_DOWN_MILLIS = "dt.application.MobileDemo.coolDownMillis";
- public static final String MAX_THROUGHPUT = "dt.application.MobileDemo.maxThroughput";
- public static final String MIN_THROUGHPUT = "dt.application.MobileDemo.minThroughput";
- private static final Logger LOG = LoggerFactory.getLogger(Application.class);
- private Range<Integer> phoneRange = Range.between(5550000, 5559999);
-
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- String lPhoneRange = conf.get(PHONE_RANGE_PROP, null);
- if (lPhoneRange != null) {
- String[] tokens = lPhoneRange.split("-");
- if (tokens.length != 2) {
- throw new IllegalArgumentException("Invalid range: " + lPhoneRange);
- }
- this.phoneRange = Range.between(Integer.parseInt(tokens[0]), Integer.parseInt(tokens[1]));
- }
- LOG.debug("Phone range {}", this.phoneRange);
-
- RandomEventGenerator phones = dag.addOperator("Receiver", RandomEventGenerator.class);
- phones.setMinvalue(this.phoneRange.getMinimum());
- phones.setMaxvalue(this.phoneRange.getMaximum());
-
- PhoneMovementGenerator movementGen = dag.addOperator("LocationFinder", PhoneMovementGenerator.class);
- dag.setAttribute(movementGen, OperatorContext.COUNTERS_AGGREGATOR, new BasicCounters.LongAggregator<MutableLong>());
-
- StatelessThroughputBasedPartitioner<PhoneMovementGenerator> partitioner = new StatelessThroughputBasedPartitioner<PhoneMovementGenerator>();
- partitioner.setCooldownMillis(conf.getLong(COOL_DOWN_MILLIS, 45000));
- partitioner.setMaximumEvents(conf.getLong(MAX_THROUGHPUT, 30000));
- partitioner.setMinimumEvents(conf.getLong(MIN_THROUGHPUT, 10000));
- dag.setAttribute(movementGen, OperatorContext.STATS_LISTENERS, Arrays.asList(new StatsListener[]{partitioner}));
- dag.setAttribute(movementGen, OperatorContext.PARTITIONER, partitioner);
-
- // generate seed numbers
- Random random = new Random();
- int maxPhone = phoneRange.getMaximum() - phoneRange.getMinimum();
- int phonesToDisplay = conf.getInt(TOTAL_SEED_NOS, 10);
- for (int i = phonesToDisplay; i-- > 0; ) {
- int phoneNo = phoneRange.getMinimum() + random.nextInt(maxPhone + 1);
- LOG.info("seed no: " + phoneNo);
- movementGen.phoneRegister.add(phoneNo);
- }
- // done generating data
- LOG.info("Finished generating seed data.");
-
- String gatewayAddress = dag.getValue(DAG.GATEWAY_CONNECT_ADDRESS);
- URI uri = URI.create("ws://" + gatewayAddress + "/pubsub");
- PubSubWebSocketOutputOperator<Object> wsOut = dag.addOperator("LocationResults", new PubSubWebSocketOutputOperator<Object>());
- wsOut.setUri(uri);
- PubSubWebSocketInputOperator<Map<String, String>> wsIn = dag.addOperator("QueryLocation", new PubSubWebSocketInputOperator<Map<String, String>>());
- wsIn.setUri(uri);
- // default partitioning: first connected stream to movementGen will be partitioned
- dag.addStream("Phone-Data", phones.integer_data, movementGen.data);
- dag.addStream("Results", movementGen.locationQueryResult, wsOut.input);
- dag.addStream("Query", wsIn.outputPort, movementGen.phoneQuery);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/mobile/src/main/java/com/datatorrent/demos/mobile/PhoneEntryOperator.java
----------------------------------------------------------------------
diff --git a/demos/mobile/src/main/java/com/datatorrent/demos/mobile/PhoneEntryOperator.java b/demos/mobile/src/main/java/com/datatorrent/demos/mobile/PhoneEntryOperator.java
deleted file mode 100644
index f6708ba..0000000
--- a/demos/mobile/src/main/java/com/datatorrent/demos/mobile/PhoneEntryOperator.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.mobile;
-
-import java.util.Map;
-import java.util.Random;
-import javax.validation.constraints.Min;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Maps;
-import com.google.common.collect.Range;
-
-import com.datatorrent.api.DefaultInputPort;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.api.annotation.InputPortFieldAnnotation;
-import com.datatorrent.common.util.BaseOperator;
-
-/**
- * Generates mobile numbers that will be displayed in mobile demo just after launch.<br></br>
- * Operator attributes:<b>
- * <ul>
- * <li>initialDisplayCount: No. of seed phone numbers that will be generated.</li>
- * <li>maxSeedPhoneNumber: The largest seed phone number.</li>
- * </ul>
- * </b>
- *
- * @since 0.3.5
- */
-public class PhoneEntryOperator extends BaseOperator
-{
- private static Logger LOG = LoggerFactory.getLogger(PhoneEntryOperator.class);
-
- private boolean seedGenerationDone = false;
-
- @Min(0)
- private int initialDisplayCount = 0;
-
- private int maxSeedPhoneNumber = 0;
- private int rangeLowerEndpoint;
- private int rangeUpperEndpoint;
-
- /**
- * Sets the initial number of phones to display on the google map.
- *
- * @param i the count of initial phone numbers to display
- */
- public void setInitialDisplayCount(int i)
- {
- initialDisplayCount = i;
- }
-
- /**
- * Sets the range for the phone numbers generated by the operator.
- *
- * @param i the range within which the phone numbers are randomly generated.
- */
- public void setPhoneRange(Range<Integer> phoneRange)
- {
- this.rangeLowerEndpoint = phoneRange.lowerEndpoint();
- this.rangeUpperEndpoint = phoneRange.upperEndpoint();
- }
-
- /**
- * Sets the max seed for random phone number generation
- *
- * @param i the number to initialize the random number phone generator.
- */
- public void setMaxSeedPhoneNumber(int number)
- {
- this.maxSeedPhoneNumber = number;
- }
-
- @InputPortFieldAnnotation(optional = true)
- public final transient DefaultInputPort<Map<String, String>> locationQuery = new DefaultInputPort<Map<String, String>>()
- {
- @Override
- public void process(Map<String, String> tuple)
- {
- seedPhones.emit(tuple);
- }
- };
-
- public final transient DefaultOutputPort<Map<String, String>> seedPhones = new DefaultOutputPort<Map<String, String>>();
-
- @Override
- public void beginWindow(long windowId)
- {
- if (!seedGenerationDone) {
- Random random = new Random();
- int maxPhone = (maxSeedPhoneNumber <= rangeUpperEndpoint && maxSeedPhoneNumber >= rangeLowerEndpoint) ? maxSeedPhoneNumber : rangeUpperEndpoint;
- maxPhone -= 5550000;
- int phonesToDisplay = initialDisplayCount > maxPhone ? maxPhone : initialDisplayCount;
- for (int i = phonesToDisplay; i-- > 0; ) {
- int phoneNo = 5550000 + random.nextInt(maxPhone + 1);
- LOG.info("seed no: " + phoneNo);
- Map<String, String> valueMap = Maps.newHashMap();
- valueMap.put(PhoneMovementGenerator.KEY_COMMAND, PhoneMovementGenerator.COMMAND_ADD);
- valueMap.put(PhoneMovementGenerator.KEY_PHONE, Integer.toString(phoneNo));
- seedPhones.emit(valueMap);
- }
- // done generating data
- seedGenerationDone = true;
- LOG.info("Finished generating seed data.");
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/mobile/src/main/java/com/datatorrent/demos/mobile/PhoneMovementGenerator.java
----------------------------------------------------------------------
diff --git a/demos/mobile/src/main/java/com/datatorrent/demos/mobile/PhoneMovementGenerator.java b/demos/mobile/src/main/java/com/datatorrent/demos/mobile/PhoneMovementGenerator.java
deleted file mode 100644
index a46e6d4..0000000
--- a/demos/mobile/src/main/java/com/datatorrent/demos/mobile/PhoneMovementGenerator.java
+++ /dev/null
@@ -1,335 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.mobile;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Random;
-import java.util.Set;
-
-import javax.validation.constraints.Min;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.commons.lang.mutable.MutableLong;
-
-import com.google.common.base.Strings;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-
-import com.datatorrent.api.Context.OperatorContext;
-import com.datatorrent.api.DefaultInputPort;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.api.annotation.InputPortFieldAnnotation;
-import com.datatorrent.common.util.BaseOperator;
-import com.datatorrent.lib.counters.BasicCounters;
-import com.datatorrent.lib.util.HighLow;
-
-/**
- * <p>
- * This operator generates the GPS locations for the phone numbers specified.
- * The range of phone numbers or a specific phone number can be set for which the GPS locations will be generated.
- * It supports querying the locations of a given phone number.
- * This is a partionable operator that can partition as the tuplesBlast increases.
- * </p>
- *
- * @since 0.3.2
- */
-public class PhoneMovementGenerator extends BaseOperator
-{
- public final transient DefaultInputPort<Integer> data = new DefaultInputPort<Integer>()
- {
- @Override
- public void process(Integer tuple)
- {
- HighLow<Integer> loc = gps.get(tuple);
- if (loc == null) {
- loc = new HighLow<Integer>(random.nextInt(range), random.nextInt(range));
- gps.put(tuple, loc);
- }
- int xloc = loc.getHigh();
- int yloc = loc.getLow();
- int state = rotate % 4;
-
- // Compute new location
- int delta = random.nextInt(100);
- if (delta >= threshold) {
- if (state < 2) {
- xloc++;
- } else {
- xloc--;
- }
- if (xloc < 0) {
- xloc += range;
- }
- }
- delta = random.nextInt(100);
- if (delta >= threshold) {
- if ((state == 1) || (state == 3)) {
- yloc++;
- } else {
- yloc--;
- }
- if (yloc < 0) {
- yloc += range;
- }
- }
- xloc %= range;
- yloc %= range;
-
- // Set new location
- HighLow<Integer> nloc = newgps.get(tuple);
- if (nloc == null) {
- newgps.put(tuple, new HighLow<Integer>(xloc, yloc));
- } else {
- nloc.setHigh(xloc);
- nloc.setLow(yloc);
- }
- rotate++;
- }
- };
-
- @InputPortFieldAnnotation(optional = true)
- public final transient DefaultInputPort<Map<String,String>> phoneQuery = new DefaultInputPort<Map<String,String>>()
- {
- @Override
- public void process(Map<String,String> tuple)
- {
- LOG.info("new query {}", tuple);
- String command = tuple.get(KEY_COMMAND);
- if (command != null) {
- if (command.equals(COMMAND_ADD)) {
- commandCounters.getCounter(CommandCounters.ADD).increment();
- String phoneStr = tuple.get(KEY_PHONE);
- registerPhone(phoneStr);
- } else if (command.equals(COMMAND_ADD_RANGE)) {
- commandCounters.getCounter(CommandCounters.ADD_RANGE).increment();
- registerPhoneRange(tuple.get(KEY_START_PHONE), tuple.get(KEY_END_PHONE));
- } else if (command.equals(COMMAND_DELETE)) {
- commandCounters.getCounter(CommandCounters.DELETE).increment();
- String phoneStr = tuple.get(KEY_PHONE);
- deregisterPhone(phoneStr);
- } else if (command.equals(COMMAND_CLEAR)) {
- commandCounters.getCounter(CommandCounters.CLEAR).increment();
- clearPhones();
- }
- }
- }
- };
-
- public static final String KEY_COMMAND = "command";
- public static final String KEY_PHONE = "phone";
- public static final String KEY_LOCATION = "location";
- public static final String KEY_REMOVED = "removed";
- public static final String KEY_START_PHONE = "startPhone";
- public static final String KEY_END_PHONE = "endPhone";
-
- public static final String COMMAND_ADD = "add";
- public static final String COMMAND_ADD_RANGE = "addRange";
- public static final String COMMAND_DELETE = "del";
- public static final String COMMAND_CLEAR = "clear";
-
- final Set<Integer> phoneRegister = Sets.newHashSet();
-
- private final transient HashMap<Integer, HighLow<Integer>> gps = new HashMap<Integer, HighLow<Integer>>();
- private final Random random = new Random();
- private int range = 50;
- private int threshold = 80;
- private int rotate = 0;
-
- protected BasicCounters<MutableLong> commandCounters;
-
- private transient OperatorContext context;
- private final transient HashMap<Integer, HighLow<Integer>> newgps = new HashMap<Integer, HighLow<Integer>>();
-
- public PhoneMovementGenerator()
- {
- this.commandCounters = new BasicCounters<MutableLong>(MutableLong.class);
- }
-
- /**
- * @return the range of the phone numbers
- */
- @Min(0)
- public int getRange()
- {
- return range;
- }
-
- /**
- * Sets the range of phone numbers for which the GPS locations need to be generated.
- *
- * @param i the range of phone numbers to set
- */
- public void setRange(int i)
- {
- range = i;
- }
-
- /**
- * @return the threshold
- */
- @Min(0)
- public int getThreshold()
- {
- return threshold;
- }
-
- /**
- * Sets the threshold that decides how frequently the GPS locations are updated.
- *
- * @param i the value that decides how frequently the GPS locations change.
- */
- public void setThreshold(int i)
- {
- threshold = i;
- }
-
- private void registerPhone(String phoneStr)
- {
- // register the phone channel
- if (Strings.isNullOrEmpty(phoneStr)) {
- return;
- }
- try {
- Integer phone = new Integer(phoneStr);
- registerSinglePhone(phone);
- } catch (NumberFormatException nfe) {
- LOG.warn("Invalid no {}", phoneStr);
- }
- }
-
- private void registerPhoneRange(String startPhoneStr, String endPhoneStr)
- {
- if (Strings.isNullOrEmpty(startPhoneStr) || Strings.isNullOrEmpty(endPhoneStr)) {
- LOG.warn("Invalid phone range {} {}", startPhoneStr, endPhoneStr);
- return;
- }
- try {
- Integer startPhone = new Integer(startPhoneStr);
- Integer endPhone = new Integer(endPhoneStr);
- if (endPhone < startPhone) {
- LOG.warn("Invalid phone range {} {}", startPhone, endPhone);
- return;
- }
- for (int i = startPhone; i <= endPhone; i++) {
- registerSinglePhone(i);
- }
- } catch (NumberFormatException nfe) {
- LOG.warn("Invalid phone range <{},{}>", startPhoneStr, endPhoneStr);
- }
- }
-
- private void registerSinglePhone(int phone)
- {
- phoneRegister.add(phone);
- LOG.debug("Registered query id with phone {}", phone);
- emitQueryResult(phone);
- }
-
- private void deregisterPhone(String phoneStr)
- {
- if (Strings.isNullOrEmpty(phoneStr)) {
- return;
- }
- try {
- Integer phone = new Integer(phoneStr);
- // remove the channel
- if (phoneRegister.contains(phone)) {
- phoneRegister.remove(phone);
- LOG.debug("Removing query id {}", phone);
- emitPhoneRemoved(phone);
- }
- } catch (NumberFormatException nfe) {
- LOG.warn("Invalid phone {}", phoneStr);
- }
- }
-
- private void clearPhones()
- {
- phoneRegister.clear();
- LOG.info("Clearing phones");
- }
-
- public final transient DefaultOutputPort<Map<String, String>> locationQueryResult = new DefaultOutputPort<Map<String, String>>();
-
- @Override
- public void setup(OperatorContext context)
- {
- this.context = context;
- commandCounters.setCounter(CommandCounters.ADD, new MutableLong());
- commandCounters.setCounter(CommandCounters.ADD_RANGE, new MutableLong());
- commandCounters.setCounter(CommandCounters.DELETE, new MutableLong());
- commandCounters.setCounter(CommandCounters.CLEAR, new MutableLong());
- }
-
- /**
- * Emit all the data and clear the hash
- */
- @Override
- public void endWindow()
- {
- for (Map.Entry<Integer, HighLow<Integer>> e: newgps.entrySet()) {
- HighLow<Integer> loc = gps.get(e.getKey());
- if (loc == null) {
- gps.put(e.getKey(), e.getValue());
- } else {
- loc.setHigh(e.getValue().getHigh());
- loc.setLow(e.getValue().getLow());
- }
- }
- boolean found = false;
- for (Integer phone: phoneRegister) {
- emitQueryResult( phone);
- found = true;
- }
- if (!found) {
- LOG.debug("No phone number");
- }
- newgps.clear();
- context.setCounters(commandCounters);
- }
-
- private void emitQueryResult(Integer phone)
- {
- HighLow<Integer> loc = gps.get(phone);
- if (loc != null) {
- Map<String, String> queryResult = new HashMap<String, String>();
- queryResult.put(KEY_PHONE, String.valueOf(phone));
- queryResult.put(KEY_LOCATION, loc.toString());
- locationQueryResult.emit(queryResult);
- }
- }
-
- private void emitPhoneRemoved(Integer phone)
- {
- Map<String,String> removedResult = Maps.newHashMap();
- removedResult.put(KEY_PHONE, String.valueOf(phone));
- removedResult.put(KEY_REMOVED,"true");
- locationQueryResult.emit(removedResult);
- }
-
- public static enum CommandCounters
- {
- ADD, ADD_RANGE, DELETE, CLEAR
- }
-
- private static final Logger LOG = LoggerFactory.getLogger(PhoneMovementGenerator.class);
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/mobile/src/main/java/com/datatorrent/demos/mobile/doc-files/Mobile.png
----------------------------------------------------------------------
diff --git a/demos/mobile/src/main/java/com/datatorrent/demos/mobile/doc-files/Mobile.png b/demos/mobile/src/main/java/com/datatorrent/demos/mobile/doc-files/Mobile.png
deleted file mode 100644
index a25da0d..0000000
Binary files a/demos/mobile/src/main/java/com/datatorrent/demos/mobile/doc-files/Mobile.png and /dev/null differ
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/mobile/src/main/java/com/datatorrent/demos/mobile/package-info.java
----------------------------------------------------------------------
diff --git a/demos/mobile/src/main/java/com/datatorrent/demos/mobile/package-info.java b/demos/mobile/src/main/java/com/datatorrent/demos/mobile/package-info.java
deleted file mode 100644
index 378e7a0..0000000
--- a/demos/mobile/src/main/java/com/datatorrent/demos/mobile/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/**
- * Mobile phones tracking demonstration application.
- */
-package com.datatorrent.demos.mobile;
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/mobile/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/demos/mobile/src/main/resources/META-INF/properties.xml b/demos/mobile/src/main/resources/META-INF/properties.xml
deleted file mode 100644
index 247bd82..0000000
--- a/demos/mobile/src/main/resources/META-INF/properties.xml
+++ /dev/null
@@ -1,82 +0,0 @@
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<configuration>
- <property>
- <name>dt.application.MobileDemo.coolDownMillis</name>
- <value>45000</value>
- </property>
- <property>
- <name>dt.application.MobileDemo.maxThroughput</name>
- <value>30000</value>
- </property>
- <property>
- <name>dt.application.MobileDemo.minThroughput</name>
- <value>10000</value>
- </property>
- <property>
- <name>dt.application.MobileDemo.operator.Receiver.tuplesBlast</name>
- <value>200</value>
- </property>
- <property>
- <name>dt.application.MobileDemo.operator.Receiver.tuplesBlastIntervalMillis</name>
- <value>5</value>
- </property>
- <property>
- <name>dt.application.MobileDemo.operator.Receiver.outputport.integer_data.attr.QUEUE_CAPACITY</name>
- <value>32768</value>
- </property>
- <property>
- <name>dt.application.MobileDemo.operator.LocationFinder.range</name>
- <value>20</value>
- </property>
- <property>
- <name>dt.application.MobileDemo.operator.LocationFinder.threshold</name>
- <value>80</value>
- </property>
- <property>
- <name>dt.application.MobileDemo.operator.LocationFinder.inputport.data.attr.QUEUE_CAPACITY</name>
- <value>32768</value>
- </property>
- <property>
- <name>dt.application.MobileDemo.operator.LocationResults.prop.topic</name>
- <value>demos.mobile.phoneLocationQueryResult</value>
- </property>
- <property>
- <name>dt.application.MobileDemo.operator.QueryLocation.prop.topic</name>
- <value>demos.mobile.phoneLocationQuery</value>
- </property>
- <property>
- <name>dt.application.MobileDemo.operator.*.attr.MEMORY_MB</name>
- <value>768</value>
- </property>
- <property>
- <name>dt.application.MobileDemo.operator.*.attr.JVM_OPTIONS</name>
- <value>-Xmx128m</value>
- </property>
- <property>
- <name>dt.application.MobileDemo.operator.*.port.*.attr.BUFFER_MEMORY_MB</name>
- <value>256</value>
- </property>
- <property>
- <name>dt.application.MobileDemo.attr.MASTER_MEMORY_MB</name>
- <value>1024</value>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/mobile/src/site/conf/my-app-conf1.xml
----------------------------------------------------------------------
diff --git a/demos/mobile/src/site/conf/my-app-conf1.xml b/demos/mobile/src/site/conf/my-app-conf1.xml
deleted file mode 100644
index f35873b..0000000
--- a/demos/mobile/src/site/conf/my-app-conf1.xml
+++ /dev/null
@@ -1,27 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<configuration>
- <property>
- <name>dt.attr.MASTER_MEMORY_MB</name>
- <value>1024</value>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/mobile/src/test/java/com/datatorrent/demos/mobile/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/demos/mobile/src/test/java/com/datatorrent/demos/mobile/ApplicationTest.java b/demos/mobile/src/test/java/com/datatorrent/demos/mobile/ApplicationTest.java
deleted file mode 100644
index 87e40bf..0000000
--- a/demos/mobile/src/test/java/com/datatorrent/demos/mobile/ApplicationTest.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.mobile;
-
-import java.net.URI;
-import java.util.HashMap;
-import java.util.Map;
-
-import javax.servlet.Servlet;
-
-import org.eclipse.jetty.server.Connector;
-import org.eclipse.jetty.server.Server;
-import org.eclipse.jetty.servlet.ServletContextHandler;
-import org.eclipse.jetty.servlet.ServletHolder;
-import org.junit.Assert;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.LocalMode;
-
-import com.datatorrent.lib.helper.SamplePubSubWebSocketServlet;
-import com.datatorrent.lib.io.PubSubWebSocketInputOperator;
-import com.datatorrent.lib.io.PubSubWebSocketOutputOperator;
-import com.datatorrent.lib.testbench.CollectorTestSink;
-
-public class ApplicationTest
-{
- private static final Logger LOG = LoggerFactory.getLogger(ApplicationTest.class);
-
- public ApplicationTest()
- {
- }
-
- /**
- * Test of getApplication method, of class Application.
- */
- @Test
- public void testGetApplication() throws Exception
- {
- Configuration conf = new Configuration(false);
- conf.addResource("dt-site-mobile.xml");
- Server server = new Server(0);
- Servlet servlet = new SamplePubSubWebSocketServlet();
- ServletHolder sh = new ServletHolder(servlet);
- ServletContextHandler contextHandler = new ServletContextHandler(server, "/", ServletContextHandler.SESSIONS);
- contextHandler.addServlet(sh, "/pubsub");
- contextHandler.addServlet(sh, "/*");
- server.start();
- Connector[] connector = server.getConnectors();
- conf.set("dt.attr.GATEWAY_CONNECT_ADDRESS", "localhost:" + connector[0].getLocalPort());
- URI uri = URI.create("ws://localhost:" + connector[0].getLocalPort() + "/pubsub");
-
- PubSubWebSocketOutputOperator<Object> outputOperator = new PubSubWebSocketOutputOperator<Object>();
- outputOperator.setUri(uri);
- outputOperator.setTopic(conf.get("dt.application.MobileDemo.operator.QueryLocation.topic"));
-
- PubSubWebSocketInputOperator<Map<String, String>> inputOperator = new PubSubWebSocketInputOperator<Map<String, String>>();
- inputOperator.setUri(uri);
- inputOperator.setTopic(conf.get("dt.application.MobileDemo.operator.LocationResults.topic"));
-
- CollectorTestSink<Object> sink = new CollectorTestSink<Object>();
- inputOperator.outputPort.setSink(sink);
-
- Map<String, String> data = new HashMap<String, String>();
- data.put("command", "add");
- data.put("phone", "5559990");
-
- Application app = new Application();
- LocalMode lma = LocalMode.newInstance();
- lma.prepareDAG(app, conf);
- LocalMode.Controller lc = lma.getController();
- lc.setHeartbeatMonitoringEnabled(false);
- lc.runAsync();
- Thread.sleep(5000);
- inputOperator.setup(null);
- outputOperator.setup(null);
- inputOperator.activate(null);
- outputOperator.beginWindow(0);
- outputOperator.input.process(data);
- outputOperator.endWindow();
- inputOperator.beginWindow(0);
- int timeoutMillis = 5000;
- while (sink.collectedTuples.size() < 5 && timeoutMillis > 0) {
- inputOperator.emitTuples();
- timeoutMillis -= 20;
- Thread.sleep(20);
- }
- inputOperator.endWindow();
- lc.shutdown();
- inputOperator.teardown();
- outputOperator.teardown();
- server.stop();
- Assert.assertTrue("size of output is 5 ", sink.collectedTuples.size() == 5);
- for (Object obj : sink.collectedTuples) {
- Assert.assertEquals("Expected phone number", "5559990", ((Map<String, String>)obj).get("phone"));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/mobile/src/test/resources/dt-site-mobile.xml
----------------------------------------------------------------------
diff --git a/demos/mobile/src/test/resources/dt-site-mobile.xml b/demos/mobile/src/test/resources/dt-site-mobile.xml
deleted file mode 100644
index 1759746..0000000
--- a/demos/mobile/src/test/resources/dt-site-mobile.xml
+++ /dev/null
@@ -1,87 +0,0 @@
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<configuration>
- <property>
- <name>dt.application.MobileDemo.class</name>
- <value>com.datatorrent.demos.mobile.Application</value>
- <description>An alias for the application</description>
- </property>
- <!--property>
- <name>dt.attr.GATEWAY_CONNECT_ADDRESS</name>
- <value>localhost:19090</value>
- </property-->
- <property>
- <name>dt.application.MobileDemo.totalSeedNumbers</name>
- <value>0</value>
- </property>
- <property>
- <name>dt.application.MobileDemo.coolDownMillis</name>
- <value>45000</value>
- </property>
- <property>
- <name>dt.application.MobileDemo.maxThroughput</name>
- <value>30000</value>
- </property>
- <property>
- <name>dt.application.MobileDemo.minThroughput</name>
- <value>1</value>
- </property>
- <property>
- <name>dt.application.MobileDemo.operator.Receiver.tuplesBlast</name>
- <value>200</value>
- </property>
- <property>
- <name>dt.application.MobileDemo.operator.Receiver.tuplesBlastIntervalMillis</name>
- <value>5</value>
- </property>
- <property>
- <name>dt.application.MobileDemo.operator.Receiver.outputport.integer_data.attr.QUEUE_CAPACITY</name>
- <value>32768</value>
- </property>
- <property>
- <name>dt.application.MobileDemo.operator.LocationFinder.range</name>
- <value>20</value>
- </property>
- <property>
- <name>dt.application.MobileDemo.operator.LocationFinder.threshold</name>
- <value>80</value>
- </property>
- <property>
- <name>dt.application.MobileDemo.operator.LocationFinder.inputport.data.attr.QUEUE_CAPACITY</name>
- <value>32768</value>
- </property>
- <property>
- <name>dt.application.MobileDemo.operator.LocationResults.topic</name>
- <value>resultTopic</value>
- </property>
- <property>
- <name>dt.application.MobileDemo.operator.QueryLocation.topic</name>
- <value>queryTopic</value>
- </property>
- <property>
- <name>dt.application.MobileDemo.operator.*.attr.MEMORY_MB</name>
- <value>2048</value>
- </property>
- <property>
- <name>dt.application.MobileDemo.attr.MASTER_MEMORY_MB</name>
- <value>1024</value>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/mobile/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/demos/mobile/src/test/resources/log4j.properties b/demos/mobile/src/test/resources/log4j.properties
deleted file mode 100644
index cf0d19e..0000000
--- a/demos/mobile/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,43 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-log4j.rootLogger=DEBUG,CONSOLE
-
-log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
-log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
-log4j.appender.CONSOLE.threshold=${test.log.console.threshold}
-test.log.console.threshold=DEBUG
-
-log4j.appender.RFA=org.apache.log4j.RollingFileAppender
-log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
-log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
-log4j.appender.RFA.File=/tmp/app.log
-
-# to enable, add SYSLOG to rootLogger
-log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender
-log4j.appender.SYSLOG.syslogHost=127.0.0.1
-log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout
-log4j.appender.SYSLOG.layout.conversionPattern=${dt.cid} %-5p [%t] %c{2} %x - %m%n
-log4j.appender.SYSLOG.Facility=LOCAL1
-
-log4j.logger.org=info
-#log4j.logger.org.apache.commons.beanutils=warn
-log4j.logger.com.datatorrent=debug
-log4j.logger.org.apache.apex=debug
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/mrmonitor/pom.xml
----------------------------------------------------------------------
diff --git a/demos/mrmonitor/pom.xml b/demos/mrmonitor/pom.xml
deleted file mode 100644
index 9373063..0000000
--- a/demos/mrmonitor/pom.xml
+++ /dev/null
@@ -1,64 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>mrmonitor</artifactId>
- <packaging>jar</packaging>
-
- <name>Apache Apex Malhar MR Monitoring Demo</name>
- <description></description>
-
- <parent>
- <groupId>org.apache.apex</groupId>
- <artifactId>malhar-demos</artifactId>
- <version>3.7.0-SNAPSHOT</version>
- </parent>
-
- <properties>
- <skipTests>true</skipTests>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-websocket</artifactId>
- <version>8.1.10.v20130312</version>
- <scope>test</scope>
- <type>jar</type>
- </dependency>
- <dependency>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-servlet</artifactId>
- <version>8.1.10.v20130312</version>
- <scope>test</scope>
- <type>jar</type>
- </dependency>
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- <version>4.3.5</version>
- <type>jar</type>
- </dependency>
- </dependencies>
-
-</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/mrmonitor/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/demos/mrmonitor/src/assemble/appPackage.xml b/demos/mrmonitor/src/assemble/appPackage.xml
deleted file mode 100644
index 4138cf2..0000000
--- a/demos/mrmonitor/src/assemble/appPackage.xml
+++ /dev/null
@@ -1,59 +0,0 @@
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
- <id>appPackage</id>
- <formats>
- <format>jar</format>
- </formats>
- <includeBaseDirectory>false</includeBaseDirectory>
- <fileSets>
- <fileSet>
- <directory>${basedir}/target/</directory>
- <outputDirectory>/app</outputDirectory>
- <includes>
- <include>${project.artifactId}-${project.version}.jar</include>
- </includes>
- </fileSet>
- <fileSet>
- <directory>${basedir}/target/deps</directory>
- <outputDirectory>/lib</outputDirectory>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/site/conf</directory>
- <outputDirectory>/conf</outputDirectory>
- <includes>
- <include>*.xml</include>
- </includes>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/main/resources/META-INF</directory>
- <outputDirectory>/META-INF</outputDirectory>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/main/resources/app</directory>
- <outputDirectory>/app</outputDirectory>
- </fileSet>
- </fileSets>
-
-</assembly>
-
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/mrmonitor/src/main/java/com/datatorrent/demos/mrmonitor/Application.java
----------------------------------------------------------------------
diff --git a/demos/mrmonitor/src/main/java/com/datatorrent/demos/mrmonitor/Application.java b/demos/mrmonitor/src/main/java/com/datatorrent/demos/mrmonitor/Application.java
deleted file mode 100644
index 5625439..0000000
--- a/demos/mrmonitor/src/main/java/com/datatorrent/demos/mrmonitor/Application.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.mrmonitor;
-
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.DAG.Locality;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-import com.datatorrent.lib.io.ConsoleOutputOperator;
-import com.datatorrent.lib.testbench.SeedEventGenerator;
-
-/**
- * Application
- *
- * @since 2.0.0
- */
-@ApplicationAnnotation(name = "MyFirstApplication")
-public class Application implements StreamingApplication
-{
-
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- // Sample DAG with 2 operators
- // Replace this code with the DAG you want to build
-
- SeedEventGenerator seedGen = dag.addOperator("seedGen", SeedEventGenerator.class);
- seedGen.setSeedStart(1);
- seedGen.setSeedEnd(10);
- seedGen.addKeyData("x", 0, 10);
- seedGen.addKeyData("y", 0, 100);
-
- ConsoleOutputOperator cons = dag.addOperator("console", new ConsoleOutputOperator());
- cons.setStringFormat("hello: %s");
-
- dag.addStream("seeddata", seedGen.val_list, cons.input).setLocality(Locality.CONTAINER_LOCAL);
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/mrmonitor/src/main/java/com/datatorrent/demos/mrmonitor/Constants.java
----------------------------------------------------------------------
diff --git a/demos/mrmonitor/src/main/java/com/datatorrent/demos/mrmonitor/Constants.java b/demos/mrmonitor/src/main/java/com/datatorrent/demos/mrmonitor/Constants.java
deleted file mode 100644
index 7930405..0000000
--- a/demos/mrmonitor/src/main/java/com/datatorrent/demos/mrmonitor/Constants.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.mrmonitor;
-
-/**
- * <p>Constants class.</p>
- *
- * @since 0.3.4
- */
-public interface Constants
-{
-
- public static final int MAX_NUMBER_OF_JOBS = 25;
-
- public static final String REDUCE_TASK_TYPE = "REDUCE";
- public static final String MAP_TASK_TYPE = "MAP";
- public static final String TASK_TYPE = "type";
- public static final String TASK_ID = "id";
-
- public static final String LEAGACY_TASK_ID = "taskId";
- public static final int MAX_TASKS = 2000;
-
- public static final String QUERY_APP_ID = "app_id";
- public static final String QUERY_JOB_ID = "job_id";
- public static final String QUERY_HADOOP_VERSION = "hadoop_version";
- public static final String QUERY_API_VERSION = "api_version";
- public static final String QUERY_RM_PORT = "rm_port";
- public static final String QUERY_HS_PORT = "hs_port";
- public static final String QUERY_HOST_NAME = "hostname";
- public static final String QUERY_KEY_COMMAND = "command";
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/mrmonitor/src/main/java/com/datatorrent/demos/mrmonitor/MRJobStatusOperator.java
----------------------------------------------------------------------
diff --git a/demos/mrmonitor/src/main/java/com/datatorrent/demos/mrmonitor/MRJobStatusOperator.java b/demos/mrmonitor/src/main/java/com/datatorrent/demos/mrmonitor/MRJobStatusOperator.java
deleted file mode 100644
index 263a1a7..0000000
--- a/demos/mrmonitor/src/main/java/com/datatorrent/demos/mrmonitor/MRJobStatusOperator.java
+++ /dev/null
@@ -1,623 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.mrmonitor;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import org.codehaus.jettison.json.JSONArray;
-import org.codehaus.jettison.json.JSONException;
-import org.codehaus.jettison.json.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.datatorrent.api.Context.OperatorContext;
-import com.datatorrent.api.DefaultInputPort;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.api.Operator;
-import com.datatorrent.api.Operator.IdleTimeHandler;
-
-import com.datatorrent.demos.mrmonitor.MRStatusObject.TaskObject;
-
-/**
- * <p>
- * MRJobStatusOperator class.
- * </p>
- *
- * @since 0.3.4
- */
-public class MRJobStatusOperator implements Operator, IdleTimeHandler
-{
- private static final Logger LOG = LoggerFactory.getLogger(MRJobStatusOperator.class);
-
- private static final String JOB_PREFIX = "job_";
- /**
- * This outputs the meta information of the job
- */
- public final transient DefaultOutputPort<String> output = new DefaultOutputPort<String>();
- /**
- * This outputs the map task information of the job
- */
- public final transient DefaultOutputPort<String> mapOutput = new DefaultOutputPort<String>();
- /**
- * This outputs the reduce task information of the job
- */
- public final transient DefaultOutputPort<String> reduceOutput = new DefaultOutputPort<String>();
- /**
- * This outputs the counter information of the job
- */
- public final transient DefaultOutputPort<String> counterOutput = new DefaultOutputPort<String>();
- /**
- * This is time in Ms before making new request for data
- */
- private transient int sleepTime = 100;
- /**
- * This is the number of consecutive windows of no change before the job is removed from map
- */
- private int maxRetrials = 10;
- /**
- * The number of minutes for which the status history of map and reduce tasks is stored
- */
- private int statusHistoryTime = 60;
- private Map<String, MRStatusObject> jobMap = new HashMap<String, MRStatusObject>();
- /**
- * This represents the maximum number of jobs the single instance of this operator is going to server at any time
- */
- private int maxJobs = Constants.MAX_NUMBER_OF_JOBS;
- private transient Iterator<MRStatusObject> iterator;
-
- /*
- * each input string is a map of the following format {"app_id":<>,"hadoop_version":<>,"api_version":<>,"command":<>,
- * "hostname":<>,"hs_port":<>,"rm_port":<>,"job_id":<>}
- */
- public final transient DefaultInputPort<MRStatusObject> input = new DefaultInputPort<MRStatusObject>()
- {
- @Override
- public void process(MRStatusObject mrStatusObj)
- {
-
- if (jobMap == null) {
- jobMap = new HashMap<String, MRStatusObject>();
- }
-
- if (jobMap.size() >= maxJobs) {
- return;
- }
-
- if ("delete".equalsIgnoreCase(mrStatusObj.getCommand())) {
- removeJob(mrStatusObj.getJobId());
- JSONObject outputJsonObject = new JSONObject();
- try {
- outputJsonObject.put("id", mrStatusObj.getJobId());
- outputJsonObject.put("removed", "true");
- output.emit(outputJsonObject.toString());
- } catch (JSONException e) {
- LOG.warn("Error creating JSON: {}", e.getMessage());
- }
- return;
- }
- if ("clear".equalsIgnoreCase(mrStatusObj.getCommand())) {
- clearMap();
- return;
- }
-
- if (jobMap.get(mrStatusObj.getJobId()) != null) {
- mrStatusObj = jobMap.get(mrStatusObj.getJobId());
- }
- if (mrStatusObj.getHadoopVersion() == 2) {
- getJsonForJob(mrStatusObj);
- } else if (mrStatusObj.getHadoopVersion() == 1) {
- getJsonForLegacyJob(mrStatusObj);
- }
- mrStatusObj.setStatusHistoryCount(statusHistoryTime);
- iterator = jobMap.values().iterator();
- emitHelper(mrStatusObj);
- }
- };
-
- public int getStatusHistoryTime()
- {
- return statusHistoryTime;
- }
-
- public void setStatusHistoryTime(int statusHistoryTime)
- {
- this.statusHistoryTime = statusHistoryTime;
- if (jobMap != null && jobMap.size() > 0) {
- for (Entry<String, MRStatusObject> entry : jobMap.entrySet()) {
- entry.getValue().setStatusHistoryCount(statusHistoryTime);
- }
- }
-
- }
-
- /**
- * This method gets the latest status of the job from the Resource Manager for jobs submitted on hadoop 2.x version
- *
- * @param statusObj
- */
- private void getJsonForJob(MRStatusObject statusObj)
- {
-
- String url = "http://" + statusObj.getUri() + ":" + statusObj.getRmPort() + "/proxy/application_" + statusObj.getAppId() + "/ws/v1/mapreduce/jobs/job_" + statusObj.getJobId();
- String responseBody = MRUtil.getJsonForURL(url);
-
- JSONObject jsonObj = MRUtil.getJsonObject(responseBody);
-
- if (jsonObj == null) {
- url = "http://" + statusObj.getUri() + ":" + statusObj.getHistoryServerPort() + "/ws/v1/history/mapreduce/jobs/job_" + statusObj.getJobId();
- responseBody = MRUtil.getJsonForURL(url);
- jsonObj = MRUtil.getJsonObject(responseBody);
- }
-
- if (jsonObj != null) {
- if (jobMap.get(statusObj.getJobId()) != null) {
- MRStatusObject tempObj = jobMap.get(statusObj.getJobId());
- if (tempObj.getJsonObject().toString().equals(jsonObj.toString())) {
- getJsonsForTasks(statusObj);
- getCounterInfoForJob(statusObj);
- return;
- }
- }
- statusObj.setModified(true);
- statusObj.setJsonObject(jsonObj);
- getCounterInfoForJob(statusObj);
- getJsonsForTasks(statusObj);
- jobMap.put(statusObj.getJobId(), statusObj);
- }
- }
-
- /**
- * This method is used to collect the metric information about the job
- *
- * @param statusObj
- */
- private void getCounterInfoForJob(MRStatusObject statusObj)
- {
- String url = "http://" + statusObj.getUri() + ":" + statusObj.getRmPort() + "/proxy/application_" + statusObj.getAppId() + "/ws/v1/mapreduce/jobs/job_" + statusObj.getJobId() + "/counters";
- String responseBody = MRUtil.getJsonForURL(url);
- JSONObject jsonObj = MRUtil.getJsonObject(responseBody);
- if (jsonObj == null) {
- url = "http://" + statusObj.getUri() + ":" + statusObj.getHistoryServerPort() + "/ws/v1/history/mapreduce/jobs/job_" + statusObj.getJobId() + "/counters";
- responseBody = MRUtil.getJsonForURL(url);
- jsonObj = MRUtil.getJsonObject(responseBody);
- }
-
- if (jsonObj != null) {
- if (statusObj.getMetricObject() == null) {
- statusObj.setMetricObject(new TaskObject(jsonObj));
- } else if (!statusObj.getMetricObject().getJsonString().equalsIgnoreCase(jsonObj.toString())) {
- statusObj.getMetricObject().setJson(jsonObj);
- statusObj.getMetricObject().setModified(true);
- }
- }
- }
-
- /**
- * This method gets the latest status of the tasks for a job from the Resource Manager for jobs submitted on hadoop
- * 2.x version
- *
- * @param statusObj
- */
- private void getJsonsForTasks(MRStatusObject statusObj)
- {
- String url = "http://" + statusObj.getUri() + ":" + statusObj.getRmPort() + "/proxy/application_" + statusObj.getAppId() + "/ws/v1/mapreduce/jobs/job_" + statusObj.getJobId() + "/tasks/";
- String responseBody = MRUtil.getJsonForURL(url);
-
- JSONObject jsonObj = MRUtil.getJsonObject(responseBody);
- if (jsonObj == null) {
- url = "http://" + statusObj.getUri() + ":" + statusObj.getHistoryServerPort() + "/ws/v1/history/mapreduce/jobs/job_" + statusObj.getJobId() + "/tasks/";
- responseBody = MRUtil.getJsonForURL(url);
-
- jsonObj = MRUtil.getJsonObject(responseBody);
- }
-
- if (jsonObj != null) {
-
- try {
- Map<String, TaskObject> mapTaskOject = statusObj.getMapJsonObject();
- Map<String, TaskObject> reduceTaskOject = statusObj.getReduceJsonObject();
- JSONArray taskJsonArray = jsonObj.getJSONObject("tasks").getJSONArray("task");
-
- for (int i = 0; i < taskJsonArray.length(); i++) {
- JSONObject taskObj = taskJsonArray.getJSONObject(i);
- if (Constants.REDUCE_TASK_TYPE.equalsIgnoreCase(taskObj.getString(Constants.TASK_TYPE))) {
- if (reduceTaskOject.get(taskObj.getString(Constants.TASK_ID)) != null) {
- TaskObject tempTaskObj = reduceTaskOject.get(taskObj.getString(Constants.TASK_ID));
- if (tempTaskObj.getJsonString().equals(taskObj.toString())) {
- continue;
- }
- tempTaskObj.setJson(taskObj);
- tempTaskObj.setModified(true);
- reduceTaskOject.put(taskObj.getString(Constants.TASK_ID), tempTaskObj);
- continue;
- }
- reduceTaskOject.put(taskObj.getString(Constants.TASK_ID), new TaskObject(taskObj));
- } else {
- if (mapTaskOject.get(taskObj.getString(Constants.TASK_ID)) != null) {
- TaskObject tempTaskObj = mapTaskOject.get(taskObj.getString(Constants.TASK_ID));
- if (tempTaskObj.getJsonString().equals(taskObj.toString())) {
- continue;
- }
- tempTaskObj.setJson(taskObj);
- tempTaskObj.setModified(true);
- mapTaskOject.put(taskObj.getString(Constants.TASK_ID), tempTaskObj);
- continue;
- }
- mapTaskOject.put(taskObj.getString(Constants.TASK_ID), new TaskObject(taskObj));
- }
- }
- statusObj.setMapJsonObject(mapTaskOject);
- statusObj.setReduceJsonObject(reduceTaskOject);
- } catch (Exception e) {
- LOG.info("exception: {}", e.getMessage());
- }
- }
-
- }
-
- /**
- * This method gets the latest status of the job from the Task Manager for jobs submitted on hadoop 1.x version
- *
- * @param statusObj
- */
- private void getJsonForLegacyJob(MRStatusObject statusObj)
- {
-
- String url = "http://" + statusObj.getUri() + ":" + statusObj.getRmPort() + "/jobdetails.jsp?format=json&jobid=job_" + statusObj.getJobId();
- String responseBody = MRUtil.getJsonForURL(url);
-
- JSONObject jsonObj = MRUtil.getJsonObject(responseBody);
- if (jsonObj == null) {
- return;
- }
-
- if (jobMap.get(statusObj.getJobId()) != null) {
- MRStatusObject tempObj = jobMap.get(statusObj.getJobId());
- if (tempObj.getJsonObject().toString().equals(jsonObj.toString())) {
- getJsonsForLegacyTasks(statusObj, "map");
- getJsonsForLegacyTasks(statusObj, "reduce");
- // output.emit(jsonObj.toString());
- // removeJob(statusObj.getJobId());
- return;
- }
- }
-
- // output.emit(jsonObj.toString());
- statusObj.setModified(true);
- statusObj.setJsonObject(jsonObj);
- getJsonsForLegacyTasks(statusObj, "map");
- getJsonsForLegacyTasks(statusObj, "reduce");
- jobMap.put(statusObj.getJobId(), statusObj);
-
- }
-
- /**
- * This method gets the latest status of the tasks for a job from the Task Manager for jobs submitted on hadoop 1.x
- * version
- *
- * @param statusObj
- * @param type
- */
- private void getJsonsForLegacyTasks(MRStatusObject statusObj, String type)
- {
- try {
- JSONObject jobJson = statusObj.getJsonObject();
- int totalTasks = ((JSONObject)((JSONObject)jobJson.get(type + "TaskSummary")).get("taskStats")).getInt("numTotalTasks");
- Map<String, TaskObject> taskMap;
- if (type.equalsIgnoreCase("map")) {
- taskMap = statusObj.getMapJsonObject();
- } else {
- taskMap = statusObj.getReduceJsonObject();
- }
-
- int totalPagenums = (totalTasks / Constants.MAX_TASKS) + 1;
- String baseUrl = "http://" + statusObj.getUri() + ":" + statusObj.getRmPort() + "/jobtasks.jsp?type=" + type + "&format=json&jobid=job_" + statusObj.getJobId() + "&pagenum=";
-
- for (int pagenum = 1; pagenum <= totalPagenums; pagenum++) {
-
- String url = baseUrl + pagenum;
- String responseBody = MRUtil.getJsonForURL(url);
-
- JSONObject jsonObj = MRUtil.getJsonObject(responseBody);
- if (jsonObj == null) {
- return;
- }
-
- JSONArray taskJsonArray = jsonObj.getJSONArray("tasksInfo");
-
- for (int i = 0; i < taskJsonArray.length(); i++) {
- JSONObject taskObj = taskJsonArray.getJSONObject(i);
- {
- if (taskMap.get(taskObj.getString(Constants.LEAGACY_TASK_ID)) != null) {
- TaskObject tempReduceObj = taskMap.get(taskObj.getString(Constants.LEAGACY_TASK_ID));
- if (tempReduceObj.getJsonString().equals(taskObj.toString())) {
- // tempReduceObj.setModified(false);
- // taskMap.put(taskObj.getString(Constants.TASK_ID), tempReduceObj);
- continue;
- }
- tempReduceObj.setJson(taskObj);
- tempReduceObj.setModified(true);
- taskMap.put(taskObj.getString(Constants.TASK_ID), tempReduceObj);
- continue;
-
- }
- taskMap.put(taskObj.getString(Constants.LEAGACY_TASK_ID), new TaskObject(taskObj));
- }
- }
- }
-
- if (type.equalsIgnoreCase("map")) {
- statusObj.setMapJsonObject(taskMap);
- } else {
- statusObj.setReduceJsonObject(taskMap);
- }
- } catch (Exception e) {
- LOG.info(e.getMessage());
- }
-
- }
-
- @Override
- public void handleIdleTime()
- {
- try {
- Thread.sleep(sleepTime);//
- } catch (InterruptedException ie) {
- // If this thread was intrrupted by nother thread
- }
- if (!iterator.hasNext()) {
- iterator = jobMap.values().iterator();
- }
-
- if (iterator.hasNext()) {
- MRStatusObject obj = iterator.next();
- if (obj.getHadoopVersion() == 2) {
- getJsonForJob(obj);
- } else if (obj.getHadoopVersion() == 1) {
- getJsonForLegacyJob(obj);
- }
- }
- }
-
- @Override
- public void setup(OperatorContext context)
- {
- iterator = jobMap.values().iterator();
- sleepTime = context.getValue(OperatorContext.SPIN_MILLIS);
- }
-
- @Override
- public void teardown()
- {
- }
-
- @Override
- public void beginWindow(long arg0)
- {
- }
-
- private void emitHelper(MRStatusObject obj)
- {
- try {
- obj.setModified(false);
- output.emit(obj.getJsonObject().toString());
- JSONObject outputJsonObject = new JSONObject();
-
- outputJsonObject.put("id", JOB_PREFIX + obj.getJobId());
- outputJsonObject.put("mapHistory", new JSONArray(obj.getMapStatusHistory()));
- outputJsonObject.put("reduceHistory", new JSONArray(obj.getReduceStatusHistory()));
- outputJsonObject.put("physicalMemoryHistory", new JSONArray(obj.getPhysicalMemeoryStatusHistory()));
- outputJsonObject.put("virtualMemoryHistory", new JSONArray(obj.getVirtualMemoryStatusHistory()));
- outputJsonObject.put("cpuHistory", new JSONArray(obj.getCpuStatusHistory()));
- output.emit(outputJsonObject.toString());
- obj.setChangedHistoryStatus(false);
-
- outputJsonObject = new JSONObject();
- outputJsonObject.put("id", JOB_PREFIX + obj.getJobId());
- JSONArray arr = new JSONArray();
-
- for (Map.Entry<String, TaskObject> mapEntry : obj.getMapJsonObject().entrySet()) {
- TaskObject json = mapEntry.getValue();
- json.setModified(false);
- arr.put(json.getJson());
- }
-
- outputJsonObject.put("tasks", arr);
- mapOutput.emit(outputJsonObject.toString());
-
- outputJsonObject = new JSONObject();
- outputJsonObject.put("id", JOB_PREFIX + obj.getJobId());
- arr = new JSONArray();
-
- for (Map.Entry<String, TaskObject> mapEntry : obj.getReduceJsonObject().entrySet()) {
- TaskObject json = mapEntry.getValue();
- json.setModified(false);
- arr.put(json.getJson());
- }
-
- outputJsonObject.put("tasks", arr);
- reduceOutput.emit(outputJsonObject.toString());
- obj.setRetrials(0);
- } catch (Exception e) {
- LOG.warn("error creating json {}", e.getMessage());
- }
-
- }
-
- @Override
- public void endWindow()
- {
- List<String> delList = new ArrayList<String>();
- try {
- for (Map.Entry<String, MRStatusObject> entry : jobMap.entrySet()) {
- MRStatusObject obj = entry.getValue();
-
- JSONObject outputJsonObject = new JSONObject();
- outputJsonObject.put("id", JOB_PREFIX + obj.getJobId());
-
- boolean modified = false;
-
- if (obj.isModified()) {
- modified = true;
- obj.setModified(false);
- output.emit(obj.getJsonObject().toString());
- if (obj.isChangedHistoryStatus()) {
- outputJsonObject.put("mapHistory", new JSONArray(obj.getMapStatusHistory()));
- outputJsonObject.put("reduceHistory", new JSONArray(obj.getReduceStatusHistory()));
- outputJsonObject.put("physicalMemoryHistory", new JSONArray(obj.getPhysicalMemeoryStatusHistory()));
- outputJsonObject.put("virtualMemoryHistory", new JSONArray(obj.getVirtualMemoryStatusHistory()));
- outputJsonObject.put("cpuHistory", new JSONArray(obj.getCpuStatusHistory()));
- output.emit(outputJsonObject.toString());
- obj.setChangedHistoryStatus(false);
- }
- }
- outputJsonObject = new JSONObject();
- outputJsonObject.put("id", JOB_PREFIX + obj.getJobId());
- JSONArray arr = new JSONArray();
-
- for (Map.Entry<String, TaskObject> mapEntry : obj.getMapJsonObject().entrySet()) {
- TaskObject json = mapEntry.getValue();
- if (json.isModified()) {
- modified = true;
- json.setModified(false);
- arr.put(json.getJson());
- }
- }
-
- if (arr.length() > 0) {
- outputJsonObject.put("tasks", arr);
- mapOutput.emit(outputJsonObject.toString());
- }
-
- outputJsonObject = new JSONObject();
- outputJsonObject.put("id", JOB_PREFIX + obj.getJobId());
- arr = new JSONArray();
-
- for (Map.Entry<String, TaskObject> mapEntry : obj.getReduceJsonObject().entrySet()) {
- TaskObject json = mapEntry.getValue();
- if (json.isModified()) {
- modified = true;
- json.setModified(false);
- arr.put(json.getJson());
- }
- }
- if (arr.length() > 0) {
- outputJsonObject.put("tasks", arr);
- reduceOutput.emit(outputJsonObject.toString());
- }
-
- if (obj.getMetricObject() != null && obj.getMetricObject().isModified()) {
- modified = true;
- obj.getMetricObject().setModified(false);
- counterOutput.emit(obj.getMetricObject().getJsonString());
- }
-
- if (!modified) {
- if (obj.getRetrials() >= maxRetrials) {
- delList.add(obj.getJobId());
- } else {
- obj.setRetrials(obj.getRetrials() + 1);
- }
- } else {
- obj.setRetrials(0);
- }
- }
- } catch (Exception ex) {
- LOG.warn("error creating json {}", ex.getMessage());
- }
-
- if (!delList.isEmpty()) {
- Iterator<String> itr = delList.iterator();
- while (itr.hasNext()) {
- removeJob(itr.next());
- }
- }
-
- }
-
- /**
- * This method removes the job from the map
- *
- * @param jobId
- */
- public void removeJob(String jobId)
- {
- if (jobMap != null) {
- jobMap.remove(jobId);
- iterator = jobMap.values().iterator();
- }
- }
-
- /**
- * This method clears the job map
- */
- public void clearMap()
- {
- if (jobMap != null) {
- jobMap.clear();
- iterator = jobMap.values().iterator();
- }
- }
-
- /**
- * This returns the maximum number of jobs the single instance of this operator is going to server at any time
- *
- * @return
- */
- public int getMaxJobs()
- {
- return maxJobs;
- }
-
- /**
- * This sets the maximum number of jobs the single instance of this operator is going to server at any time
- *
- * @param maxJobs
- */
- public void setMaxJobs(int maxJobs)
- {
- this.maxJobs = maxJobs;
- }
-
- /**
- * This sets the number of consecutive windows of no change before the job is removed from map
- *
- * @return
- */
- public int getMaxRetrials()
- {
- return maxRetrials;
- }
-
- /**
- * This returns the number of consecutive windows of no change before the job is removed from map
- *
- * @param maxRetrials
- */
- public void setMaxRetrials(int maxRetrials)
- {
- this.maxRetrials = maxRetrials;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/mrmonitor/src/main/java/com/datatorrent/demos/mrmonitor/MRMonitoringApplication.java
----------------------------------------------------------------------
diff --git a/demos/mrmonitor/src/main/java/com/datatorrent/demos/mrmonitor/MRMonitoringApplication.java b/demos/mrmonitor/src/main/java/com/datatorrent/demos/mrmonitor/MRMonitoringApplication.java
deleted file mode 100644
index 037378a..0000000
--- a/demos/mrmonitor/src/main/java/com/datatorrent/demos/mrmonitor/MRMonitoringApplication.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.mrmonitor;
-
-import java.net.URI;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-
-import com.datatorrent.lib.io.PubSubWebSocketInputOperator;
-import com.datatorrent.lib.io.PubSubWebSocketOutputOperator;
-
-/**
- * <p>
- * MRDebuggerApplication class.
- * </p>
- *
- * @since 0.3.4
- */
-@ApplicationAnnotation(name = "MRMonitoringDemo")
-public class MRMonitoringApplication implements StreamingApplication
-{
-
- private static final Logger logger = LoggerFactory.getLogger(MRMonitoringApplication.class);
-
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- String daemonAddress = dag.getValue(DAG.GATEWAY_CONNECT_ADDRESS);
- MRJobStatusOperator mrJobOperator = dag.addOperator("JobMonitor", new MRJobStatusOperator());
- URI uri = URI.create("ws://" + daemonAddress + "/pubsub");
- logger.info("WebSocket with daemon at {}", daemonAddress);
-
- PubSubWebSocketInputOperator wsIn = dag.addOperator("Query", new PubSubWebSocketInputOperator());
- wsIn.setUri(uri);
-
- MapToMRObjectOperator queryConverter = dag.addOperator("QueryConverter", new MapToMRObjectOperator());
-
- /**
- * This is used to emit the meta data about the job
- */
- PubSubWebSocketOutputOperator<Object> wsOut = dag.addOperator("JobOutput", new PubSubWebSocketOutputOperator<Object>());
- wsOut.setUri(uri);
-
- /**
- * This is used to emit the information of map tasks of the job
- */
- PubSubWebSocketOutputOperator<Object> wsMapOut = dag.addOperator("MapJob", new PubSubWebSocketOutputOperator<Object>());
- wsMapOut.setUri(uri);
-
- /**
- * This is used to emit the information of reduce tasks of the job
- */
- PubSubWebSocketOutputOperator<Object> wsReduceOut = dag.addOperator("ReduceJob", new PubSubWebSocketOutputOperator<Object>());
- wsReduceOut.setUri(uri);
-
- /**
- * This is used to emit the metric information of the job
- */
- PubSubWebSocketOutputOperator<Object> wsCounterOut = dag.addOperator("JobCounter", new PubSubWebSocketOutputOperator<Object>());
- wsCounterOut.setUri(uri);
-
- dag.addStream("QueryConversion", wsIn.outputPort, queryConverter.input);
- dag.addStream("QueryProcessing", queryConverter.output, mrJobOperator.input);
- dag.addStream("JobData", mrJobOperator.output, wsOut.input);
- dag.addStream("MapData", mrJobOperator.mapOutput, wsMapOut.input);
- dag.addStream("ReduceData", mrJobOperator.reduceOutput, wsReduceOut.input);
- dag.addStream("CounterData", mrJobOperator.counterOutput, wsCounterOut.input);
- }
-
-}
[23/30] apex-malhar git commit: Renamed demos to examples. Packages
and artifactid names are changed as suggested.
Posted by th...@apache.org.
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/Application.java
----------------------------------------------------------------------
diff --git a/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/Application.java b/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/Application.java
deleted file mode 100644
index 55b299f..0000000
--- a/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/Application.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.machinedata;
-
-import java.util.Map;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-
-import com.datatorrent.contrib.redis.RedisKeyValPairOutputOperator;
-import com.datatorrent.demos.machinedata.data.MachineKey;
-import com.datatorrent.demos.machinedata.operator.MachineInfoAveragingOperator;
-import com.datatorrent.demos.machinedata.operator.MachineInfoAveragingPrerequisitesOperator;
-import com.datatorrent.lib.io.SmtpOutputOperator;
-
-/**
- * <p>
- * Resource monitor application.
- * </p>
- *
- * @since 0.3.5
- */
-@ApplicationAnnotation(name = "MachineDataDemo")
-@SuppressWarnings("unused")
-public class Application implements StreamingApplication
-{
-
- private static final Logger LOG = LoggerFactory.getLogger(Application.class);
-
- /**
- * This function sets up the DAG for calculating the average
- *
- * @param dag the DAG instance
- * @param conf the configuration instance
- * @return MachineInfoAveragingPrerequisitesOperator
- */
- private MachineInfoAveragingPrerequisitesOperator addAverageCalculation(DAG dag, Configuration conf)
- {
- MachineInfoAveragingPrerequisitesOperator prereqAverageOper = dag.addOperator("Aggregator", MachineInfoAveragingPrerequisitesOperator.class);
- MachineInfoAveragingOperator averageOperator = dag.addOperator("AverageCalculator", MachineInfoAveragingOperator.class);
- RedisKeyValPairOutputOperator<MachineKey, Map<String, String>> redisAvgOperator = dag.addOperator("Persister", new RedisKeyValPairOutputOperator<MachineKey, Map<String, String>>());
- dag.addStream("Average", averageOperator.outputPort, redisAvgOperator.input);
- SmtpOutputOperator smtpOutputOperator = dag.addOperator("Alerter", new SmtpOutputOperator());
- dag.addStream("Aggregates", prereqAverageOper.outputPort, averageOperator.inputPort);
- dag.addStream("Alerts", averageOperator.smtpAlert, smtpOutputOperator.input);
- return prereqAverageOper;
- }
-
- /**
- * Create the DAG
- */
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- InputReceiver randomGen = dag.addOperator("Receiver", InputReceiver.class);
- DimensionGenerator dimensionGenerator = dag.addOperator("DimensionsGenerator", DimensionGenerator.class);
- dag.addStream("Events", randomGen.outputInline, dimensionGenerator.inputPort);
- MachineInfoAveragingPrerequisitesOperator prereqAverageOper = addAverageCalculation(dag, conf);
- dag.addStream("DimensionalData", dimensionGenerator.outputInline, prereqAverageOper.inputPort);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/DimensionGenerator.java
----------------------------------------------------------------------
diff --git a/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/DimensionGenerator.java b/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/DimensionGenerator.java
deleted file mode 100644
index 75c2a02..0000000
--- a/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/DimensionGenerator.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.machinedata;
-
-import com.datatorrent.api.Context;
-import com.datatorrent.api.DefaultInputPort;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.common.util.BaseOperator;
-import com.datatorrent.demos.machinedata.data.MachineInfo;
-import com.datatorrent.demos.machinedata.data.MachineKey;
-
-/**
- * <p>
- * Information tuple generator with randomness.
- * </p>
- *
- * @since 0.3.5
- */
-@SuppressWarnings("unused")
-public class DimensionGenerator extends BaseOperator
-{
- public transient DefaultOutputPort<MachineInfo> outputInline = new DefaultOutputPort<>();
- public transient DefaultOutputPort<MachineInfo> output = new DefaultOutputPort<>();
- private int threshold = 90;
-
- public final transient DefaultInputPort<MachineInfo> inputPort = new DefaultInputPort<MachineInfo>()
- {
-
- @Override
- public void process(MachineInfo tuple)
- {
- emitDimensions(tuple);
- }
-
- };
-
- @Override
- public void setup(Context.OperatorContext context)
- {
- super.setup(context);
- }
-
- /**
- * This returns the threshold value set
- * @return
- */
- public int getThreshold()
- {
- return threshold;
- }
-
- /**
- * This function sets the threshold value. This value is used to check the maximum value for cpu/ram/hdd
- * @param threshold
- */
- public void setThreshold(int threshold)
- {
- this.threshold = threshold;
- }
-
- /**
- * This function takes in the tuple from upstream operator and generates tuples with different dimension combinations
- *
- * @param tuple
- */
- private void emitDimensions(MachineInfo tuple)
- {
- MachineKey tupleKey = tuple.getMachineKey();
-
- for (int i = 0; i < 64; i++) {
- MachineKey machineKey = new MachineKey(tupleKey.getTimeKey(),tupleKey.getDay());
- if ((i & 1) != 0) {
- machineKey.setCustomer(tupleKey.getCustomer());
- }
- if ((i & 2) != 0) {
- machineKey.setProduct(tupleKey.getProduct());
- }
- if ((i & 4) != 0) {
- machineKey.setOs(tupleKey.getOs());
- }
- if ((i & 8) != 0) {
- machineKey.setDeviceId(tupleKey.getDeviceId());
- }
- if ((i & 16) != 0) {
- machineKey.setSoftware1(tupleKey.getSoftware1());
- }
- if ((i & 32) != 0) {
- machineKey.setSoftware2(tupleKey.getSoftware2());
- }
-
- int cpu = tuple.getCpu();
- int ram = tuple.getRam();
- int hdd = tuple.getHdd();
- MachineInfo machineInfo = new MachineInfo();
- machineInfo.setMachineKey(machineKey);
- machineInfo.setCpu((cpu < threshold) ? cpu : threshold);
- machineInfo.setRam((ram < threshold) ? ram : threshold);
- machineInfo.setHdd((hdd < threshold) ? hdd : threshold);
- outputInline.emit(machineInfo);
- output.emit(machineInfo);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/InputReceiver.java
----------------------------------------------------------------------
diff --git a/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/InputReceiver.java b/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/InputReceiver.java
deleted file mode 100644
index 85ec954..0000000
--- a/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/InputReceiver.java
+++ /dev/null
@@ -1,523 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.machinedata;
-
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
-import java.util.Calendar;
-import java.util.Date;
-import java.util.Random;
-import java.util.TimeZone;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.datatorrent.api.Context;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.api.InputOperator;
-import com.datatorrent.common.util.BaseOperator;
-import com.datatorrent.demos.machinedata.data.MachineInfo;
-import com.datatorrent.demos.machinedata.data.MachineKey;
-
-/**
- * <p>
- * Information tuple generator with randomness.
- * </p>
- *
- * @since 0.3.5
- */
-@SuppressWarnings("unused")
-public class InputReceiver extends BaseOperator implements InputOperator
-{
- private static final Logger logger = LoggerFactory.getLogger(InputReceiver.class);
-
- public transient DefaultOutputPort<MachineInfo> outputInline = new DefaultOutputPort<>();
- private final Random randomGen = new Random();
-
- private int customerMin = 1;
- private int customerMax = 5;
- private int productMin = 4;
- private int productMax = 6;
- private int osMin = 10;
- private int osMax = 12;
- private int software1Min = 10;
- private int software1Max = 12;
- private int software2Min = 12;
- private int software2Max = 14;
- private int software3Min = 4;
- private int software3Max = 6;
-
- private int deviceIdMin = 1;
- private int deviceIdMax = 50;
-
- private int tupleBlastSize = 1001;
-
- private static final DateFormat minuteDateFormat = new SimpleDateFormat("HHmm");
- private static final DateFormat dayDateFormat = new SimpleDateFormat("d");
-
- static {
- TimeZone tz = TimeZone.getTimeZone("GMT");
- minuteDateFormat.setTimeZone(tz);
- dayDateFormat.setTimeZone(tz);
-
- }
-
- @Override
- public void setup(Context.OperatorContext context)
- {
- super.setup(context);
- }
-
- @Override
- public void beginWindow(long windowId)
- {
- super.beginWindow(windowId);
- }
-
- @Override
- public void emitTuples()
- {
- int count = 0;
- Calendar calendar = Calendar.getInstance();
- Date date = calendar.getTime();
- String timeKey = minuteDateFormat.format(date);
- String day = dayDateFormat.format(date);
-
- while (count < tupleBlastSize) {
- randomGen.setSeed(System.currentTimeMillis());
-
- int customerVal = genCustomerId();
- int productVal = genProductVer();
- int osVal = genOsVer();
- int software1Val = genSoftware1Ver();
- int software2Val = genSoftware2Ver();
- int software3Val = genSoftware3Ver();
- int deviceIdVal = genDeviceId();
-
- int cpuVal = genCpu(calendar);
- int ramVal = genRam(calendar);
- int hddVal = genHdd(calendar);
-
- MachineKey machineKey = new MachineKey(timeKey, day);
-
- machineKey.setCustomer(customerVal);
- machineKey.setProduct(productVal);
- machineKey.setOs(osVal);
- machineKey.setDeviceId(deviceIdVal);
- machineKey.setSoftware1(software1Val);
- machineKey.setSoftware2(software2Val);
- machineKey.setSoftware3(software3Val);
- MachineInfo machineInfo = new MachineInfo();
- machineInfo.setMachineKey(machineKey);
- machineInfo.setCpu(cpuVal);
- machineInfo.setRam(ramVal);
- machineInfo.setHdd(hddVal);
-
- outputInline.emit(machineInfo);
-
- count++;
- }
- }
-
- private int genCustomerId()
- {
- int range = customerMax - customerMin + 1;
- return customerMin + randomGen.nextInt(range);
- }
-
- private int genProductVer()
- {
- int range = productMax - productMin + 1;
- return productMin + randomGen.nextInt(range);
- }
-
- private int genOsVer()
- {
- int range = osMax - osMin + 1;
- return osMin + randomGen.nextInt(range);
- }
-
- private int genSoftware3Ver()
- {
- int range = software3Max - software3Min + 1;
- return software3Min + randomGen.nextInt(range);
- }
-
- private int genDeviceId()
- {
- int range = deviceIdMax - deviceIdMin + 1;
- return deviceIdMin + randomGen.nextInt(range);
- }
-
- private int genSoftware1Ver()
- {
- int range = software1Max - software1Min + 1;
- return software1Min + randomGen.nextInt(range);
- }
-
- private int genSoftware2Ver()
- {
- int range = software2Max - software2Min + 1;
- return software2Min + randomGen.nextInt(range);
- }
-
- private int genCpu(Calendar cal)
- {
- int minute = cal.get(Calendar.MINUTE);
- int second;
- int range = minute / 2 + 19;
- if (minute / 17 == 0) {
- second = cal.get(Calendar.SECOND);
- return (30 + randomGen.nextInt(range) + (minute % 7) - (second % 11));
- } else if (minute / 47 == 0) {
- second = cal.get(Calendar.SECOND);
- return (7 + randomGen.nextInt(range) + (minute % 7) - (second % 7));
- } else {
- second = cal.get(Calendar.SECOND);
- return (randomGen.nextInt(range) + (minute % 19) + (second % 7));
- }
- }
-
- private int genRam(Calendar cal)
- {
- int minute = cal.get(Calendar.MINUTE);
- int second;
- int range = minute + 1;
- if (minute / 23 == 0) {
- second = cal.get(Calendar.SECOND);
- return (20 + randomGen.nextInt(range) + (minute % 5) - (second % 11));
- } else if (minute / 37 == 0) {
- second = cal.get(Calendar.SECOND);
- return (11 + randomGen.nextInt(60) - (minute % 5) - (second % 11));
- } else {
- second = cal.get(Calendar.SECOND);
- return (randomGen.nextInt(range) + (minute % 17) + (second % 11));
- }
- }
-
- private int genHdd(Calendar cal)
- {
- int minute = cal.get(Calendar.MINUTE);
- int second;
- int range = minute / 2 + 1;
- if (minute / 37 == 0) {
- second = cal.get(Calendar.SECOND);
- return (25 + randomGen.nextInt(range) - minute % 7 - second % 11);
- } else {
- second = cal.get(Calendar.SECOND);
- return (randomGen.nextInt(range) + minute % 23 + second % 11);
- }
- }
-
- /**
- * This method returns the minimum value for customer
- *
- * @return
- */
- public int getCustomerMin()
- {
- return customerMin;
- }
-
- /**
- * This method is used to set the minimum value for customer
- *
- * @param customerMin the minimum customer value
- */
- public void setCustomerMin(int customerMin)
- {
- this.customerMin = customerMin;
- }
-
- /**
- * This method returns the max value for customer
- *
- * @return
- */
- public int getCustomerMax()
- {
- return customerMax;
- }
-
- /**
- * This method is used to set the max value for customer
- *
- * @param customerMax the max customer value
- */
- public void setCustomerMax(int customerMax)
- {
- this.customerMax = customerMax;
- }
-
- /**
- * This method returns the minimum value for product
- *
- * @return
- */
- public int getProductMin()
- {
- return productMin;
- }
-
- /**
- * This method is used to set the minimum value for product
- *
- * @param productMin the minimum product value
- */
- public void setProductMin(int productMin)
- {
- this.productMin = productMin;
- }
-
- /**
- * This method returns the max value for product
- *
- * @return
- */
- public int getProductMax()
- {
- return productMax;
- }
-
- /**
- * This method is used to set the max value for product
- *
- * @param productMax the max product value
- */
- public void setProductMax(int productMax)
- {
- this.productMax = productMax;
- }
-
- /**
- * This method returns the minimum value for OS
- *
- * @return
- */
- public int getOsMin()
- {
- return osMin;
- }
-
- /**
- * This method is used to set the minimum value for OS
- *
- * @param osMin the min OS value
- */
- public void setOsMin(int osMin)
- {
- this.osMin = osMin;
- }
-
- /**
- * This method returns the max value for OS
- *
- * @return
- */
- public int getOsMax()
- {
- return osMax;
- }
-
- /**
- * This method is used to set the max value for OS
- *
- * @param osMax the max OS value
- */
- public void setOsMax(int osMax)
- {
- this.osMax = osMax;
- }
-
- /**
- * This method returns the minimum value for software1
- *
- * @return
- */
- public int getSoftware1Min()
- {
- return software1Min;
- }
-
- /**
- * This method is used to set the minimum value for software1
- *
- * @param software1Min the minimum software1 value
- */
- public void setSoftware1Min(int software1Min)
- {
- this.software1Min = software1Min;
- }
-
- /**
- * This method returns the max value for software1
- *
- * @return
- */
- public int getSoftware1Max()
- {
- return software1Max;
- }
-
- /**
- * This method is used to set the max value for software1
- *
- * @param software1Max the max software1 value
- */
- public void setSoftware1Max(int software1Max)
- {
- this.software1Max = software1Max;
- }
-
- /**
- * This method returns the minimum value for software2
- *
- * @return
- */
- public int getSoftware2Min()
- {
- return software2Min;
- }
-
- /**
- * This method is used to set the minimum value for software2
- *
- * @param software2Min the minimum software2 value
- */
- public void setSoftware2Min(int software2Min)
- {
- this.software2Min = software2Min;
- }
-
- /**
- * This method returns the max value for software2
- *
- * @return
- */
- public int getSoftware2Max()
- {
- return software2Max;
- }
-
- /**
- * This method is used to set the max value for software2
- *
- * @param software2Max the max software2 value
- */
- public void setSoftware2Max(int software2Max)
- {
- this.software2Max = software2Max;
- }
-
- /**
- * This method returns the minimum value for software3
- *
- * @return
- */
- public int getSoftware3Min()
- {
- return software3Min;
- }
-
- /**
- * This method is used to set the minimum value for software3
- *
- * @param software3Min the minimum software3 value
- */
- public void setSoftware3Min(int software3Min)
- {
- this.software3Min = software3Min;
- }
-
- /**
- * This method returns the max value for software3
- *
- * @return
- */
- public int getSoftware3Max()
- {
- return software3Max;
- }
-
- /**
- * This method is used to set the max value for software3
- *
- * @param software3Max the max software3 value
- */
- public void setSoftware3Max(int software3Max)
- {
- this.software3Max = software3Max;
- }
-
- /**
- * This method returns the minimum value for deviceId
- *
- * @return
- */
- public int getDeviceIdMin()
- {
- return deviceIdMin;
- }
-
- /**
- * This method is used to set the minimum value for deviceId
- *
- * @param deviceIdMin the minimum deviceId value
- */
- public void setDeviceIdMin(int deviceIdMin)
- {
- this.deviceIdMin = deviceIdMin;
- }
-
- /**
- * This method returns the max value for deviceId
- *
- * @return
- */
- public int getDeviceIdMax()
- {
- return deviceIdMax;
- }
-
- /**
- * This method is used to set the max value for deviceId
- *
- * @param deviceIdMax the max deviceId value
- */
- public void setDeviceIdMax(int deviceIdMax)
- {
- this.deviceIdMax = deviceIdMax;
- }
-
- /**
- * @return the tupleBlastSize
- */
- public int getTupleBlastSize()
- {
- return tupleBlastSize;
- }
-
- /**
- * @param tupleBlastSize the tupleBlastSize to set
- */
- public void setTupleBlastSize(int tupleBlastSize)
- {
- this.tupleBlastSize = tupleBlastSize;
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/data/AverageData.java
----------------------------------------------------------------------
diff --git a/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/data/AverageData.java b/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/data/AverageData.java
deleted file mode 100644
index 3c74cc5..0000000
--- a/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/data/AverageData.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.machinedata.data;
-
-
-/**
- * This class stores the value of sum and the count of values summed.
- * <p>
- * AverageData class.
- * </p>
- *
- * @since 0.3.5
- */
-public class AverageData
-{
-
- private long cpu;
- private long hdd;
- private long ram;
- private long count;
-
- /**
- * This is default constructor that sets the sum and count to 0
- */
- public AverageData()
- {
-
- }
-
- /**
- * This constructor takes the value of sum and count and initialize the local attributes to corresponding values
- *
- * @param count
- * the value of count
- */
- public AverageData(long cpu,long hdd,long ram, long count)
- {
- this.cpu = cpu;
- this.ram = ram;
- this.hdd = hdd;
- this.count = count;
- }
-
- public long getCpu()
- {
- return cpu;
- }
-
- public void setCpu(long cpu)
- {
- this.cpu = cpu;
- }
-
- public long getHdd()
- {
- return hdd;
- }
-
- public void setHdd(long hdd)
- {
- this.hdd = hdd;
- }
-
- public long getRam()
- {
- return ram;
- }
-
- public void setRam(long ram)
- {
- this.ram = ram;
- }
-
- /**
- * This returns the value of count
- * @return
- */
- public long getCount()
- {
- return count;
- }
-
- /**
- * This method sets the value of count
- * @param count
- */
- public void setCount(long count)
- {
- this.count = count;
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/data/MachineInfo.java
----------------------------------------------------------------------
diff --git a/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/data/MachineInfo.java b/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/data/MachineInfo.java
deleted file mode 100644
index 6f02a24..0000000
--- a/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/data/MachineInfo.java
+++ /dev/null
@@ -1,158 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.machinedata.data;
-
-/**
- * This class stores the cpu% usage, ram% usage, hdd% usage and key information about a particular machine
- * <p>
- * MachineInfo class.
- * </p>
- *
- * @since 0.3.5
- */
-public class MachineInfo
-{
- private MachineKey machineKey;
- private int cpu;
- private int ram;
- private int hdd;
-
- /**
- * This default constructor
- */
- public MachineInfo()
- {
- }
-
- /**
- * This constructor takes MachineKey as input and initialize local attributes
- *
- * @param machineKey
- * the MachineKey instance
- */
- public MachineInfo(MachineKey machineKey)
- {
- this.machineKey = machineKey;
- }
-
- /**
- * This constructor takes MachineKey, cpu usage, ram usage, hdd usage as input and initialize local attributes
- *
- * @param machineKey
- * the MachineKey instance
- * @param cpu
- * the CPU% usage
- * @param ram
- * the RAM% usage
- * @param hdd
- * the HDD% usage
- */
- public MachineInfo(MachineKey machineKey, int cpu, int ram, int hdd)
- {
- this.machineKey = machineKey;
- this.cpu = cpu;
- this.ram = ram;
- this.hdd = hdd;
- }
-
- /**
- * This method returns the MachineKey
- *
- * @return
- */
- public MachineKey getMachineKey()
- {
- return machineKey;
- }
-
- /**
- * This method sets the MachineKey
- *
- * @param machineKey
- * the MachineKey instance
- */
- public void setMachineKey(MachineKey machineKey)
- {
- this.machineKey = machineKey;
- }
-
- /**
- * This method returns the CPU% usage
- *
- * @return
- */
- public int getCpu()
- {
- return cpu;
- }
-
- /**
- * This method sets the CPU% usage
- *
- * @param cpu
- * the CPU% usage
- */
- public void setCpu(int cpu)
- {
- this.cpu = cpu;
- }
-
- /**
- * This method returns the RAM% usage
- *
- * @return
- */
- public int getRam()
- {
- return ram;
- }
-
- /**
- * This method sets the RAM% usage
- *
- * @param ram
- * the RAM% usage
- */
- public void setRam(int ram)
- {
- this.ram = ram;
- }
-
- /**
- * This method returns the HDD% usage
- *
- * @return
- */
- public int getHdd()
- {
- return hdd;
- }
-
- /**
- * This method sets the HDD% usage
- *
- * @param hdd
- * the HDD% usage
- */
- public void setHdd(int hdd)
- {
- this.hdd = hdd;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/data/MachineKey.java
----------------------------------------------------------------------
diff --git a/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/data/MachineKey.java b/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/data/MachineKey.java
deleted file mode 100644
index 2b3bb1c..0000000
--- a/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/data/MachineKey.java
+++ /dev/null
@@ -1,381 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.machinedata.data;
-
-/**
- * This class stores the information about various softwares, deviceIds, OS of the device
- * <p>
- * MachineKey class.
- * </p>
- *
- * @since 0.3.5
- */
-public class MachineKey
-{
-
- private Integer customer;
- private Integer product;
- private Integer os;
- private Integer software1;
- private Integer software2;
- private Integer software3;
- private Integer deviceId;
- private String timeKey;
- private String day;
-
- /**
- * This constructor takes the format in which time has to be captured and the day when this instance is created
- *
- * @param timeKey the format in which time has to be captured
- * @param day the day when this instance is created
- */
- public MachineKey(String timeKey, String day)
- {
- this.timeKey = timeKey;
- this.day = day;
- }
-
- /**
- * This is default constructor
- */
- public MachineKey()
- {
- }
-
- /**
- * This constructor takes format in which time has to be captured, the day when this instance is created, the customer
- * id, product Id on the device, OS version on the device, software1 version on the device, software2 version on the device,
- * software3 version on the device, deviceId on the device,
- *
- * @param timeKey the format in which time has to be captured
- * @param day the day when this instance is created
- * @param customer the customer Id
- * @param product product Id
- * @param os OS version
- * @param software1 software1 version
- * @param software2 software2 version
- * @param software3 software3 version
- * @param deviceId deviceId
- */
- public MachineKey(String timeKey, String day, Integer customer, Integer product, Integer os, Integer software1, Integer software2, Integer software3, Integer deviceId)
- {
- this.timeKey = timeKey;
- this.day = day;
- this.customer = customer;
- this.product = product;
- this.os = os;
- this.software1 = software1;
- this.software2 = software2;
- this.software3 = software3;
- this.deviceId = deviceId;
- }
-
- /**
- * This method returns the format in which the time is captured. The time is the time when this instance of MachineKey
- * was generated. For e.g. HHmm to capture Hour and minute
- *
- * @return
- */
- public String getTimeKey()
- {
- return timeKey;
- }
-
- /**
- * This method sets the format in which the time is captured. The time is the time when this instance of MachineKey
- * was generated. For e.g. HHmm to capture Hour and minute
- *
- * @param timeKey
- * the value of format
- */
- public void setTimeKey(String timeKey)
- {
- this.timeKey = timeKey;
- }
-
- /**
- * This method returns the day of the month when this instance of MachineKey was generated
- *
- * @return
- */
- public String getDay()
- {
- return day;
- }
-
- /**
- * This method sets the day of the month when this instance of MachineKey was generated
- *
- * @param day
- * the day of the month
- */
- public void setDay(String day)
- {
- this.day = day;
- }
-
- /**
- * This method returns the customer Id
- *
- * @return
- */
- public Integer getCustomer()
- {
- return customer;
- }
-
- /**
- * This method sets the customer Id
- *
- * @param customer
- * the customer Id
- */
- public void setCustomer(Integer customer)
- {
- this.customer = customer;
- }
-
- /**
- * This method returns product on the device
- *
- * @return
- */
- public Integer getProduct()
- {
- return product;
- }
-
- /**
- * This method sets the product on the device
- *
- * @param product
- * the value of product
- */
- public void setProduct(Integer product)
- {
- this.product = product;
- }
-
- /**
- * This method returns the OS version on the device
- *
- * @return
- */
- public Integer getOs()
- {
- return os;
- }
-
- /**
- * This method sets the OS version on the device
- *
- * @param os
- * OS version
- */
- public void setOs(Integer os)
- {
- this.os = os;
- }
-
- /**
- * This method returns the version of the software1 on the device
- *
- * @return
- */
- public Integer getSoftware1()
- {
- return software1;
- }
-
- /**
- * This method sets the version of the software1 on the device
- *
- * @param software1 the version of the software1
- */
- public void setSoftware1(Integer software1)
- {
- this.software1 = software1;
- }
-
- /**
- * This method returns the version of the software2 on the device
- *
- * @return
- */
- public Integer getSoftware2()
- {
- return software2;
- }
-
- /**
- * This method sets the version of the software2 on the device
- *
- * @param software2
- * the version of the software2
- */
- public void setSoftware2(Integer software2)
- {
- this.software2 = software2;
- }
-
- /**
- * This method returns the version of the software3 on the device
- *
- * @return
- */
- public Integer getSoftware3()
- {
- return software3;
- }
-
- /**
- * This method sets the version of the software3 on the device
- *
- * @param software3
- * the version of the software3
- */
- public void setSoftware3(Integer software3)
- {
- this.software3 = software3;
- }
-
- @Override
- public int hashCode()
- {
- int key = 0;
- if (customer != null) {
- key |= (1 << 31);
- key ^= customer;
- }
- if (product != null) {
- key |= (1 << 30);
- key ^= product;
- }
- if (os != null) {
- key |= (1 << 29);
- key ^= os;
- }
- if (software1 != null) {
- key |= (1 << 28);
- key ^= software1;
- }
- if (software2 != null) {
- key |= (1 << 27);
- key ^= software2;
- }
- if (software3 != null) {
- key |= (1 << 26);
- key ^= software3;
- }
- if (deviceId != null) {
- key |= (1 << 25);
- key ^= deviceId;
- }
- if (timeKey != null) {
- key |= (1 << 24);
- key ^= timeKey.hashCode();
- }
- if (day != null) {
- key |= (1 << 23);
- key ^= day.hashCode();
- }
-
- return key;
- }
-
- @Override
- public boolean equals(Object obj)
- {
- if (!(obj instanceof MachineKey)) {
- return false;
- }
- MachineKey mkey = (MachineKey)obj;
- return checkStringEqual(this.timeKey, mkey.timeKey) && checkStringEqual(this.day, mkey.day) && checkIntEqual(this.customer, mkey.customer) && checkIntEqual(this.product, mkey.product) && checkIntEqual(this.os, mkey.os) && checkIntEqual(this.software1, mkey.software1) && checkIntEqual(this.software2, mkey.software2) && checkIntEqual(this.software3, mkey.software3) && checkIntEqual(this.deviceId, mkey.deviceId);
- }
-
- private boolean checkIntEqual(Integer a, Integer b)
- {
- if ((a == null) && (b == null)) {
- return true;
- }
- if ((a != null) && a.equals(b)) {
- return true;
- }
- return false;
- }
-
- private boolean checkStringEqual(String a, String b)
- {
- if ((a == null) && (b == null)) {
- return true;
- }
- if ((a != null) && a.equals(b)) {
- return true;
- }
- return false;
- }
-
- @Override
- public String toString()
- {
- StringBuilder sb = new StringBuilder(timeKey);
- if (customer != null) {
- sb.append("|0:").append(customer);
- }
- if (product != null) {
- sb.append("|1:").append(product);
- }
- if (os != null) {
- sb.append("|2:").append(os);
- }
- if (software1 != null) {
- sb.append("|3:").append(software1);
- }
- if (software2 != null) {
- sb.append("|4:").append(software2);
- }
- if (software3 != null) {
- sb.append("|5:").append(software3);
- }
- if (deviceId != null) {
- sb.append("|6:").append(deviceId);
- }
- return sb.toString();
- }
-
- /**
- * This method returns the deviceId of the device
- * @return The deviceId
- */
- public Integer getDeviceId()
- {
- return deviceId;
- }
-
- /**
- * This method sets the deviceId of the device
- *
- * @param deviceId
- */
- public void setDeviceId(Integer deviceId)
- {
- this.deviceId = deviceId;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/data/ResourceType.java
----------------------------------------------------------------------
diff --git a/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/data/ResourceType.java b/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/data/ResourceType.java
deleted file mode 100644
index d474c5c..0000000
--- a/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/data/ResourceType.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.machinedata.data;
-
-import java.util.Map;
-
-import com.google.common.collect.Maps;
-
-/**
- * This class captures the resources whose usage is collected for each device
- * <p>ResourceType class.</p>
- *
- * @since 0.3.5
- */
-public enum ResourceType
-{
-
- CPU("cpu"), RAM("ram"), HDD("hdd");
-
- private static Map<String, ResourceType> descToResource = Maps.newHashMap();
-
- static {
- for (ResourceType type : ResourceType.values()) {
- descToResource.put(type.desc, type);
- }
- }
-
- private String desc;
-
- private ResourceType(String desc)
- {
- this.desc = desc;
- }
-
- @Override
- public String toString()
- {
- return desc;
- }
-
- /**
- * This method returns ResourceType for the given description
- * @param desc the description
- * @return
- */
- public static ResourceType getResourceTypeOf(String desc)
- {
- return descToResource.get(desc);
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/operator/CalculatorOperator.java
----------------------------------------------------------------------
diff --git a/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/operator/CalculatorOperator.java b/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/operator/CalculatorOperator.java
deleted file mode 100644
index 8f68dab..0000000
--- a/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/operator/CalculatorOperator.java
+++ /dev/null
@@ -1,277 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.machinedata.operator;
-
-import java.io.Serializable;
-import java.math.BigDecimal;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-
-import javax.validation.constraints.Max;
-import javax.validation.constraints.Min;
-
-import com.google.common.base.Objects;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-
-import com.datatorrent.api.DefaultInputPort;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.api.StreamCodec;
-import com.datatorrent.common.util.BaseOperator;
-import com.datatorrent.demos.machinedata.data.MachineInfo;
-import com.datatorrent.demos.machinedata.data.MachineKey;
-import com.datatorrent.demos.machinedata.data.ResourceType;
-import com.datatorrent.demos.machinedata.util.DataTable;
-import com.datatorrent.lib.codec.KryoSerializableStreamCodec;
-import com.datatorrent.lib.util.KeyValPair;
-
-/**
- * <p>
- * CalculatorOperator class.
- * </p>
- *
- * @since 0.3.5
- */
-public class CalculatorOperator extends BaseOperator
-{
-
- private final DataTable<MachineKey, ResourceType, List<Integer>> data = new DataTable<>();
-
- @Min(1)
- @Max(99)
- private int kthPercentile = 95; // kth percentile
- private boolean computePercentile;
- private boolean computeSD;
- private boolean computeMax;
-
- private int percentileThreshold = 80;
- private int sdThreshold = 70;
- private int maxThreshold = 99;
-
- public final transient DefaultInputPort<MachineInfo> dataPort = new DefaultInputPort<MachineInfo>()
- {
- @Override
- public void process(MachineInfo tuple)
- {
- addDataToCache(tuple);
- }
-
- /**
- * Stream codec used for partitioning.
- */
- @Override
- public StreamCodec<MachineInfo> getStreamCodec()
- {
- return new MachineInfoStreamCodec();
- }
- };
-
- public final transient DefaultOutputPort<KeyValPair<MachineKey, Map<ResourceType, Double>>> percentileOutputPort = new DefaultOutputPort<>();
-
- public final transient DefaultOutputPort<KeyValPair<MachineKey, Map<ResourceType, Double>>> sdOutputPort = new DefaultOutputPort<>();
-
- public final transient DefaultOutputPort<KeyValPair<MachineKey, Map<ResourceType, Integer>>> maxOutputPort = new DefaultOutputPort<>();
-
- public transient DefaultOutputPort<String> smtpAlert = new DefaultOutputPort<>();
-
- private void addDataToCache(MachineInfo tuple)
- {
- MachineKey machineKey = tuple.getMachineKey();
- if (!data.containsRow(machineKey)) {
- data.put(machineKey, ResourceType.CPU, Lists.<Integer>newArrayList());
- data.put(machineKey, ResourceType.RAM, Lists.<Integer>newArrayList());
- data.put(machineKey, ResourceType.HDD, Lists.<Integer>newArrayList());
- }
- data.get(machineKey, ResourceType.CPU).add(tuple.getCpu());
- data.get(machineKey, ResourceType.RAM).add(tuple.getRam());
- data.get(machineKey, ResourceType.HDD).add(tuple.getHdd());
- }
-
- @Override
- public void endWindow()
- {
-
- if (computePercentile) {
- for (MachineKey machineKey : data.rowKeySet()) {
- Collections.sort(data.get(machineKey, ResourceType.CPU));
- Collections.sort(data.get(machineKey, ResourceType.RAM));
- Collections.sort(data.get(machineKey, ResourceType.HDD));
-
- Map<ResourceType, Double> percentileData = Maps.newHashMap();
- percentileData.put(ResourceType.CPU, getKthPercentile(data.get(machineKey, ResourceType.CPU)));
- percentileData.put(ResourceType.RAM, getKthPercentile(data.get(machineKey, ResourceType.RAM)));
- percentileData.put(ResourceType.HDD, getKthPercentile(data.get(machineKey, ResourceType.HDD)));
- percentileOutputPort.emit(new KeyValPair<>(machineKey, percentileData));
-
- for (ResourceType resourceType : percentileData.keySet()) {
- double percentileValue = percentileData.get(resourceType);
- if (percentileValue > percentileThreshold) {
- emitAlert(resourceType, machineKey, percentileValue, "Percentile");
- }
- }
- }
- }
- if (computeSD) {
- for (MachineKey machineKey : data.rowKeySet()) {
-
- Map<ResourceType, Double> sdData = Maps.newHashMap();
-
- for (ResourceType resourceType : ResourceType.values()) {
- sdData.put(resourceType, getSD(data.get(machineKey, resourceType)));
- }
- sdOutputPort.emit(new KeyValPair<>(machineKey, sdData));
-
- for (ResourceType resourceType : sdData.keySet()) {
- double sdValue = sdData.get(resourceType);
- if (sdValue > sdThreshold) {
- emitAlert(resourceType, machineKey, sdValue, "SD");
- }
- }
- }
- }
- if (computeMax) {
- for (MachineKey machineKey : data.rowKeySet()) {
-
- Map<ResourceType, Integer> maxData = Maps.newHashMap();
- maxData.put(ResourceType.CPU, Collections.max(data.get(machineKey, ResourceType.CPU)));
- maxData.put(ResourceType.RAM, Collections.max(data.get(machineKey, ResourceType.RAM)));
- maxData.put(ResourceType.HDD, Collections.max(data.get(machineKey, ResourceType.HDD)));
-
- maxOutputPort.emit(new KeyValPair<>(machineKey, maxData));
-
- for (ResourceType resourceType : maxData.keySet()) {
- double sdValue = maxData.get(resourceType).doubleValue();
- if (sdValue > maxThreshold) {
- emitAlert(resourceType, machineKey, sdValue, "Max");
- }
- }
- }
- }
- data.clear();
- }
-
- private void emitAlert(ResourceType type, MachineKey machineKey, double alertVal, String prefix)
- {
- BigDecimal decimalVal = new BigDecimal(alertVal);
- decimalVal = decimalVal.setScale(2, BigDecimal.ROUND_HALF_UP);
- String alertTime = machineKey.getDay() + machineKey.getTimeKey();
- smtpAlert.emit(prefix + "-" + type.toString().toUpperCase() + " alert at " + alertTime + " " + type + " usage breached current usage: " + decimalVal.doubleValue() + "% threshold: " + percentileThreshold + "%\n\n" + machineKey);
- }
-
- private double getKthPercentile(List<Integer> sorted)
- {
-
- double val = (kthPercentile * sorted.size()) / 100.0;
- if (val == (int)val) {
- // Whole number
- int idx = (int)val - 1;
- return (sorted.get(idx) + sorted.get(idx + 1)) / 2.0;
- } else {
- int idx = (int)Math.round(val) - 1;
- return sorted.get(idx);
- }
- }
-
- private double getSD(List<Integer> data)
- {
- int sum = 0;
- for (int i : data) {
- sum += i;
- }
- double avg = sum / (data.size() * 1.0);
- double sd = 0;
- for (Integer point : data) {
- sd += Math.pow(point - avg, 2);
- }
- return Math.sqrt(sd);
- }
-
- /**
- * @param kVal the percentile which will be emitted by this operator
- */
- public void setKthPercentile(int kVal)
- {
- this.kthPercentile = kVal;
- }
-
- /**
- * @param doCompute when true percentile will be computed
- */
- public void setComputePercentile(boolean doCompute)
- {
- this.computePercentile = doCompute;
- }
-
- /**
- * @param doCompute when true standard deviation will be computed
- */
- public void setComputeSD(boolean doCompute)
- {
- this.computeSD = doCompute;
- }
-
- /**
- * @param doCompute when true max will be computed
- */
- public void setComputeMax(boolean doCompute)
- {
- this.computeMax = doCompute;
- }
-
- /**
- * @param threshold for percentile when breached will cause alert
- */
- public void setPercentileThreshold(int threshold)
- {
- this.percentileThreshold = threshold;
- }
-
- /**
- * @param threshold for standard deviation when breached will cause alert
- */
- public void setSDThreshold(int threshold)
- {
- this.sdThreshold = threshold;
- }
-
- /**
- * @param threshold for Max when breached will cause alert
- */
- public void setMaxThreshold(int threshold)
- {
- this.maxThreshold = threshold;
- }
-
- public static class MachineInfoStreamCodec extends KryoSerializableStreamCodec<MachineInfo> implements Serializable
- {
- public MachineInfoStreamCodec()
- {
- super();
- }
-
- @Override
- public int getPartition(MachineInfo o)
- {
- return Objects.hashCode(o.getMachineKey().getCustomer(), o.getMachineKey().getOs(), o.getMachineKey().getProduct(), o.getMachineKey().getSoftware1(), o.getMachineKey().getSoftware2(), o.getMachineKey().getSoftware3());
- }
-
- private static final long serialVersionUID = 201411031403L;
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/operator/MachineInfoAveragingOperator.java
----------------------------------------------------------------------
diff --git a/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/operator/MachineInfoAveragingOperator.java b/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/operator/MachineInfoAveragingOperator.java
deleted file mode 100644
index bbfd547..0000000
--- a/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/operator/MachineInfoAveragingOperator.java
+++ /dev/null
@@ -1,215 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.machinedata.operator;
-
-import java.math.BigDecimal;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
-import java.util.Calendar;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.Map;
-
-import com.google.common.collect.Maps;
-
-import com.datatorrent.api.DefaultInputPort;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.common.util.BaseOperator;
-
-import com.datatorrent.demos.machinedata.data.AverageData;
-import com.datatorrent.demos.machinedata.data.MachineInfo;
-import com.datatorrent.demos.machinedata.data.MachineKey;
-import com.datatorrent.lib.util.KeyHashValPair;
-import com.datatorrent.lib.util.KeyValPair;
-
-/**
- * This class calculates the average for various resources across different devices for a given key
- * <p>MachineInfoAveragingOperator class.</p>
- *
- * @since 0.9.0
- */
-@SuppressWarnings("unused")
-public class MachineInfoAveragingOperator extends BaseOperator
-{
-
- public static final String CPU = "cpu";
- public static final String RAM = "ram";
- public static final String HDD = "hdd";
- public static final String DAY = "day";
-
- private final transient Map<MachineKey, AverageData> dataMap = new HashMap<>();
-
- public final transient DefaultOutputPort<KeyValPair<MachineKey, Map<String, String>>> outputPort = new DefaultOutputPort<>();
-
- public transient DefaultOutputPort<String> smtpAlert = new DefaultOutputPort<>();
-
- private int threshold = 95;
-
- /**
- * Buffer all the tuples as is till end window gets called
- */
- public final transient DefaultInputPort<KeyHashValPair<MachineKey, AverageData>> inputPort = new DefaultInputPort<KeyHashValPair<MachineKey, AverageData>>()
- {
-
- @Override
- public void process(KeyHashValPair<MachineKey, AverageData> tuple)
- {
- addTuple(tuple);
- }
- };
-
- /**
- * This method returns the threshold value
- *
- * @return
- */
- public int getThreshold()
- {
- return threshold;
- }
-
- /**
- * This method sets the threshold value. If the average usage for any Resource is above this for a given key, then the alert is sent
- *
- * @param threshold the threshold value
- */
- public void setThreshold(int threshold)
- {
- this.threshold = threshold;
- }
-
- /**
- * This adds the given tuple to the dataMap
- *
- * @param tuple input tuple
- */
- private void addTuple(KeyHashValPair<MachineKey, AverageData> tuple)
- {
- MachineKey key = tuple.getKey();
- dataMap.put(key, tuple.getValue());
- }
-
- @Override
- public void endWindow()
- {
- for (Map.Entry<MachineKey, AverageData> entry : dataMap.entrySet()) {
- MachineKey key = entry.getKey();
- AverageData averageResultMap = entry.getValue();
- Map<String, String> averageResult = Maps.newHashMap();
- long count = averageResultMap.getCount();
- double average = averageResultMap.getCpu() / count;
- averageResult.put(CPU, average + "");
- emitAlert(average, CPU, key);
- average = averageResultMap.getHdd() / count;
- averageResult.put(HDD, average + "");
- emitAlert(average, HDD, key);
- average = averageResultMap.getRam() / count;
- averageResult.put(RAM, average + "");
- emitAlert(average, RAM, key);
- averageResult.put(DAY, key.getDay());
- outputPort.emit(new KeyValPair<>(key, averageResult));
- }
- dataMap.clear();
- }
-
- private void emitAlert(double average, String resourceType, MachineKey key)
- {
- if (average > threshold) {
- BigDecimal bd = new BigDecimal(average);
- bd = bd.setScale(2, BigDecimal.ROUND_HALF_UP);
- String stime = key.getDay() + key.getTimeKey();
- String skey = getKeyInfo(key);
- smtpAlert.emit(resourceType.toUpperCase() + " alert at " + stime + " " + resourceType + " usage breached current usage: " + bd.doubleValue() + "% threshold: " + threshold + "%\n\n" + skey);
- }
- }
-
- /**
- * This method is used to artificially generate alerts
- *
- * @param genAlert
- */
- public void setGenAlert(boolean genAlert)
- {
- Calendar calendar = Calendar.getInstance();
- long timestamp = System.currentTimeMillis();
- calendar.setTimeInMillis(timestamp);
- DateFormat minuteDateFormat = new SimpleDateFormat("HHmm");
- Date date = calendar.getTime();
- String timeKey = minuteDateFormat.format(date);
- DateFormat dayDateFormat = new SimpleDateFormat("dd");
- String day = dayDateFormat.format(date);
-
- MachineKey alertKey = new MachineKey(timeKey, day);
- alertKey.setCustomer(1);
- alertKey.setProduct(5);
- alertKey.setOs(10);
- alertKey.setSoftware1(12);
- alertKey.setSoftware2(14);
- alertKey.setSoftware3(6);
-
- MachineInfo machineInfo = new MachineInfo();
- machineInfo.setMachineKey(alertKey);
- machineInfo.setCpu(threshold + 1);
- machineInfo.setRam(threshold + 1);
- machineInfo.setHdd(threshold + 1);
-
- smtpAlert.emit("CPU Alert: CPU Usage threshold (" + threshold + ") breached: current % usage: " + getKeyInfo(alertKey));
- smtpAlert.emit("RAM Alert: RAM Usage threshold (" + threshold + ") breached: current % usage: " + getKeyInfo(alertKey));
- smtpAlert.emit("HDD Alert: HDD Usage threshold (" + threshold + ") breached: current % usage: " + getKeyInfo(alertKey));
- }
-
- /**
- * This method returns the String for a given MachineKey instance
- *
- * @param key MachineKey instance that needs to be converted to string
- * @return
- */
- private String getKeyInfo(MachineKey key)
- {
- StringBuilder sb = new StringBuilder();
- if (key instanceof MachineKey) {
- MachineKey mkey = (MachineKey)key;
- Integer customer = mkey.getCustomer();
- if (customer != null) {
- sb.append("customer: " + customer + "\n");
- }
- Integer product = mkey.getProduct();
- if (product != null) {
- sb.append("product version: " + product + "\n");
- }
- Integer os = mkey.getOs();
- if (os != null) {
- sb.append("os version: " + os + "\n");
- }
- Integer software1 = mkey.getSoftware1();
- if (software1 != null) {
- sb.append("software1 version: " + software1 + "\n");
- }
- Integer software2 = mkey.getSoftware2();
- if (software2 != null) {
- sb.append("software2 version: " + software2 + "\n");
- }
- Integer software3 = mkey.getSoftware3();
- if (software3 != null) {
- sb.append("software3 version: " + software3 + "\n");
- }
- }
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/operator/MachineInfoAveragingPrerequisitesOperator.java
----------------------------------------------------------------------
diff --git a/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/operator/MachineInfoAveragingPrerequisitesOperator.java b/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/operator/MachineInfoAveragingPrerequisitesOperator.java
deleted file mode 100644
index cb5fa5a..0000000
--- a/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/operator/MachineInfoAveragingPrerequisitesOperator.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.machinedata.operator;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import com.datatorrent.api.DefaultInputPort;
-import com.datatorrent.api.DefaultOutputPort;
-
-import com.datatorrent.common.util.BaseOperator;
-import com.datatorrent.demos.machinedata.data.AverageData;
-import com.datatorrent.demos.machinedata.data.MachineInfo;
-import com.datatorrent.demos.machinedata.data.MachineKey;
-import com.datatorrent.lib.util.KeyHashValPair;
-
-/**
- * This class calculates the partial sum and count for tuples generated by upstream Operator
- * <p> MachineInfoAveragingPrerequisitesOperator class. </p>
- *
- * @since 0.3.5
- */
-public class MachineInfoAveragingPrerequisitesOperator extends BaseOperator
-{
-
- // Aggregate sum of all values seen for a key.
- private Map<MachineKey, AverageData> sums = new HashMap<>();
-
- public final transient DefaultOutputPort<KeyHashValPair<MachineKey, AverageData>> outputPort = new DefaultOutputPort<KeyHashValPair<MachineKey, AverageData>>()
- {
- @Override
- public Unifier<KeyHashValPair<MachineKey, AverageData>> getUnifier()
- {
- MachineInfoAveragingUnifier unifier = new MachineInfoAveragingUnifier();
- return unifier;
- }
- };
-
- public transient DefaultInputPort<MachineInfo> inputPort = new DefaultInputPort<MachineInfo>()
- {
-
- @Override
- public void process(MachineInfo tuple)
- {
- MachineKey key = tuple.getMachineKey();
- AverageData averageData = sums.get(key);
- if (averageData == null) {
- averageData = new AverageData(tuple.getCpu(), tuple.getHdd(), tuple.getRam(), 1);
- sums.put(key, averageData);
- } else {
- averageData.setCpu(averageData.getCpu() + tuple.getCpu());
- averageData.setRam(averageData.getRam() + tuple.getRam());
- averageData.setHdd(averageData.getHdd() + tuple.getHdd());
- averageData.setCount(averageData.getCount() + 1);
- }
- }
- };
-
- @Override
- public void endWindow()
- {
-
- for (Map.Entry<MachineKey, AverageData> entry : sums.entrySet()) {
- if (outputPort.isConnected()) {
- outputPort.emit(new KeyHashValPair<>(entry.getKey(), entry.getValue()));
- }
- }
- sums.clear();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/operator/MachineInfoAveragingUnifier.java
----------------------------------------------------------------------
diff --git a/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/operator/MachineInfoAveragingUnifier.java b/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/operator/MachineInfoAveragingUnifier.java
deleted file mode 100644
index e0b67f3..0000000
--- a/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/operator/MachineInfoAveragingUnifier.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.machinedata.operator;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import com.datatorrent.api.Context.OperatorContext;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.api.Operator.Unifier;
-
-import com.datatorrent.demos.machinedata.data.AverageData;
-import com.datatorrent.demos.machinedata.data.MachineKey;
-import com.datatorrent.lib.util.KeyHashValPair;
-
-/**
- * This class calculates the partial sum and count for a given key
- * <p>MachineInfoAveragingUnifier class.</p>
- *
- * @since 0.9.0
- */
-public class MachineInfoAveragingUnifier implements Unifier<KeyHashValPair<MachineKey, AverageData>>
-{
-
- private Map<MachineKey, AverageData> sums = new HashMap<>();
- public final transient DefaultOutputPort<KeyHashValPair<MachineKey, AverageData>> outputPort = new DefaultOutputPort<>();
-
- @Override
- public void beginWindow(long arg0)
- {
- // TODO Auto-generated method stub
-
- }
-
- @Override
- public void endWindow()
- {
- for (Map.Entry<MachineKey, AverageData> entry : sums.entrySet()) {
- outputPort.emit(new KeyHashValPair<>(entry.getKey(), entry.getValue()));
- }
- sums.clear();
-
- }
-
- @Override
- public void setup(OperatorContext arg0)
- {
- // TODO Auto-generated method stub
-
- }
-
- @Override
- public void teardown()
- {
- // TODO Auto-generated method stub
-
- }
-
- @Override
- public void process(KeyHashValPair<MachineKey, AverageData> arg0)
- {
- MachineKey tupleKey = arg0.getKey();
- AverageData averageData = sums.get(tupleKey);
- AverageData tupleValue = arg0.getValue();
- if (averageData == null) {
- sums.put(tupleKey, tupleValue);
- } else {
- averageData.setCpu(averageData.getCpu() + tupleValue.getCpu());
- averageData.setRam(averageData.getRam() + tupleValue.getRam());
- averageData.setHdd(averageData.getHdd() + tupleValue.getHdd());
- averageData.setCount(averageData.getCount() + tupleValue.getCount());
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/util/Combinatorics.java
----------------------------------------------------------------------
diff --git a/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/util/Combinatorics.java b/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/util/Combinatorics.java
deleted file mode 100644
index 6c4256a..0000000
--- a/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/util/Combinatorics.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.machinedata.util;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Generate combinations of elements for the given array of elements.
- *
- * Implements nCr = n! / (r! * (n-r)!)
- *
- * @since 0.3.5
- */
-public class Combinatorics<T>
-{
-
- private T[] values;
- private int size = -1;
- private List<T> result;
- private Map<Integer, List<T>> resultMap = new HashMap<Integer, List<T>>();
- private int resultMapSize = 0;
-
- /**
- * Generates all possible combinations with all the sizes.
- *
- * @param values
- */
- public Combinatorics(T[] values)
- {
- this.values = values;
- this.size = -1;
- this.result = new ArrayList<>();
- }
-
- /**
- * Generates all possible combinations with the given size.
- *
- * @param values
- * @param size
- */
- public Combinatorics(T[] values, int size)
- {
- this.values = values;
- this.size = size;
- this.result = new ArrayList<>();
- }
-
- public Map<Integer, List<T>> generate()
- {
-
- if (size == -1) {
- size = values.length;
- for (int i = 1; i <= size; i++) {
- int[] tmp = new int[i];
- Arrays.fill(tmp, -1);
- generateCombinations(0, 0, tmp);
- }
- } else {
- int[] tmp = new int[size];
- Arrays.fill(tmp, -1);
- generateCombinations(0, 0, tmp);
- }
- return resultMap;
- }
-
- public void generateCombinations(int start, int depth, int[] tmp)
- {
- if (depth == tmp.length) {
- for (int j = 0; j < depth; j++) {
- result.add(values[tmp[j]]);
- }
- resultMap.put(++resultMapSize, result);
- result = new ArrayList<>();
- return;
- }
- for (int i = start; i < values.length; i++) {
- tmp[depth] = i;
- generateCombinations(i + 1, depth + 1, tmp);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/util/DataTable.java
----------------------------------------------------------------------
diff --git a/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/util/DataTable.java b/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/util/DataTable.java
deleted file mode 100644
index f8f2d33..0000000
--- a/demos/machinedata/src/main/java/com/datatorrent/demos/machinedata/util/DataTable.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.machinedata.util;
-
-import java.util.Map;
-import java.util.Set;
-
-import javax.annotation.Nullable;
-
-import com.google.common.collect.Maps;
-
-/**
- * <p>DataTable class.</p>
- *
- * @since 0.3.5
- */
-public class DataTable<R,C,E>
-{
-
- //machineKey, [cpu,ram,hdd] -> value
- private final Map<R,Map<C,E>> table = Maps.newHashMap();
-
- public boolean containsRow(R rowKey)
- {
- return table.containsKey(rowKey);
- }
-
- public void put(R rowKey,C colKey, E entry)
- {
- if (!containsRow(rowKey)) {
- table.put(rowKey, Maps.<C,E>newHashMap());
- }
- table.get(rowKey).put(colKey, entry);
- }
-
- @Nullable
- public E get(R rowKey, C colKey)
- {
- if (!containsRow(rowKey)) {
- return null;
- }
- return table.get(rowKey).get(colKey);
- }
-
- public Set<R> rowKeySet()
- {
- return table.keySet();
- }
-
- public void clear()
- {
- table.clear();
- }
-
- public Map<R,Map<C,E>> getTable()
- {
- return table;
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/machinedata/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/demos/machinedata/src/main/resources/META-INF/properties.xml b/demos/machinedata/src/main/resources/META-INF/properties.xml
deleted file mode 100644
index afe8783..0000000
--- a/demos/machinedata/src/main/resources/META-INF/properties.xml
+++ /dev/null
@@ -1,139 +0,0 @@
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<configuration>
- <property>
- <name>dt.attr.STREAMING_WINDOW_SIZE_MILLIS</name>
- <value>1000</value>
- </property>
- <property>
- <name>dt.application.MachineDataDemo.operator.Receiver.attr.APPLICATION_WINDOW_COUNT
- </name>
- <value>5</value>
- </property>
- <property>
- <name>dt.application.MachineDataDemo.operator.Receiver.attr.PARTITIONER</name>
- <value>com.datatorrent.common.partitioner.StatelessPartitioner:1</value>
- </property>
- <property>
- <name>dt.application.MachineDataDemo.operator.DimensionsGenerator.attr.APPLICATION_WINDOW_COUNT
- </name>
- <value>5</value>
- </property>
- <property>
- <name>dt.application.MachineDataDemo.stream.Events.locality
- </name>
- <value>CONTAINER_LOCAL</value>
- </property>
- <property>
- <name>dt.application.MachineDataDemo.operator.DimensionsGenerator.inputport.inputPort.attr.PARTITION_PARALLEL
- </name>
- <value>true</value>
- </property>
- <property>
- <name>dt.application.MachineDataDemo.operator.Aggregator.inputport.inputPort.attr.PARTITION_PARALLEL
- </name>
- <value>true</value>
- </property>
- <property>
- <name>dt.application.MachineDataDemo.operator.Aggregator.attr.APPLICATION_WINDOW_COUNT
- </name>
- <value>5</value>
- </property>
- <property>
- <name>dt.application.MachineDataDemo.operator.AverageCalculator.attr.APPLICATION_WINDOW_COUNT
- </name>
- <value>5</value>
- </property>
- <property>
- <name>dt.application.MachineDataDemo.operator.Persister.inputport.input.attr.PARTITION_PARALLEL
- </name>
- <value>true</value>
- </property>
- <property>
- <name>dt.application.MachineDataDemo.operator.Persister.attr.APPLICATION_WINDOW_COUNT
- </name>
- <value>5</value>
- </property>
- <property>
- <name>dt.application.MachineDataDemo.operator.Persister.store.dbIndex
- </name>
- <value>2</value>
- </property>
- <property>
- <name>dt.application.MachineDataDemo.operator.Persister.store.host
- </name>
- <value></value>
- </property>
- <property>
- <name>dt.application.MachineDataDemo.operator.Persister.store.port
- </name>
- <value>6379</value>
- </property>
- <property>
- <name>dt.application.MachineDataDemo.port.*.attr.QUEUE_CAPACITY</name>
- <value>32000</value>
- </property>
- <property>
- <name>dt.application.MachineDataDemo.operator.Alerter.from
- </name>
- <value></value>
- </property>
- <property>
- <name>dt.application.MachineDataDemo.operator.Alerter.subject
- </name>
- <value>Alert!!!</value>
- </property>
- <property>
- <name>dt.application.MachineDataDemo.operator.Alerter.recipients.TO
- </name>
- <value></value>
- </property>
- <property>
- <name>dt.application.MachineDataDemo.operator.Alerter.content
- </name>
- <value>{}</value>
- </property>
- <property>
- <name>dt.application.MachineDataDemo.operator.Alerter.smtpHost
- </name>
- <value>localhost</value>
- </property>
- <property>
- <name>dt.application.MachineDataDemo.operator.Alerter.smtpPort
- </name>
- <value>25</value>
- </property>
- <property>
- <name>dt.application.MachineDataDemo.operator.Alerter.useSsl
- </name>
- <value>false</value>
- </property>
- <property>
- <name>dt.application.MachineDataDemo.operator.Aggregator.outputport.outputPort.attr.UNIFIER_LIMIT
- </name>
- <value>8</value>
- </property>
- <property>
- <name>dt.application.MachineDataDemo.stream.DimensionalData.locality
- </name>
- <value>THREAD_LOCAL</value>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/machinedata/src/site/conf/my-app-conf1.xml
----------------------------------------------------------------------
diff --git a/demos/machinedata/src/site/conf/my-app-conf1.xml b/demos/machinedata/src/site/conf/my-app-conf1.xml
deleted file mode 100644
index f35873b..0000000
--- a/demos/machinedata/src/site/conf/my-app-conf1.xml
+++ /dev/null
@@ -1,27 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<configuration>
- <property>
- <name>dt.attr.MASTER_MEMORY_MB</name>
- <value>1024</value>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/machinedata/src/test/java/com/datatorrent/demos/machinedata/CalculatorOperatorTest.java
----------------------------------------------------------------------
diff --git a/demos/machinedata/src/test/java/com/datatorrent/demos/machinedata/CalculatorOperatorTest.java b/demos/machinedata/src/test/java/com/datatorrent/demos/machinedata/CalculatorOperatorTest.java
deleted file mode 100644
index 0e397be..0000000
--- a/demos/machinedata/src/test/java/com/datatorrent/demos/machinedata/CalculatorOperatorTest.java
+++ /dev/null
@@ -1,196 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.machinedata;
-
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
-import java.util.Calendar;
-import java.util.Date;
-import java.util.List;
-import java.util.Map;
-
-import org.junit.Assert;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.ImmutableList;
-
-import com.datatorrent.demos.machinedata.data.MachineInfo;
-import com.datatorrent.demos.machinedata.data.MachineKey;
-import com.datatorrent.demos.machinedata.data.ResourceType;
-import com.datatorrent.demos.machinedata.operator.CalculatorOperator;
-import com.datatorrent.lib.testbench.CollectorTestSink;
-import com.datatorrent.lib.util.KeyValPair;
-import com.datatorrent.lib.util.TimeBucketKey;
-
-/**
- * @since 0.3.5
- */
-public class CalculatorOperatorTest
-{
- private static DateFormat minuteDateFormat = new SimpleDateFormat("HHmm");
- private static Logger LOG = LoggerFactory.getLogger(CalculatorOperatorTest.class);
-
- /**
- * Test node logic emits correct results
- */
- @Test
- public void testNodeProcessing() throws Exception
- {
- CalculatorOperator calculatorOperator = new CalculatorOperator();
- calculatorOperator.setup(null);
-
- calculatorOperator.setComputePercentile(true);
- calculatorOperator.setComputeMax(true);
- calculatorOperator.setComputeSD(true);
-
- testPercentile(calculatorOperator);
- }
-
- public void testPercentile(CalculatorOperator oper)
- {
-
- CollectorTestSink sortSink = new CollectorTestSink();
- oper.percentileOutputPort.setSink(sortSink);
- oper.setKthPercentile(50);
- Calendar calendar = Calendar.getInstance();
- Date date = calendar.getTime();
- String timeKey = minuteDateFormat.format(date);
- String day = calendar.get(Calendar.DAY_OF_MONTH) + "";
-
- Integer vs = new Integer(1);
- MachineKey mk = new MachineKey(timeKey, day, vs, vs, vs, vs, vs, vs, vs);
-
- oper.beginWindow(0);
-
- MachineInfo info = new MachineInfo(mk, 1, 1, 1);
- oper.dataPort.process(info);
-
- info.setCpu(2);
- oper.dataPort.process(info);
-
- info.setCpu(3);
- oper.dataPort.process(info);
-
- oper.endWindow();
-
- Assert.assertEquals("number emitted tuples", 1, sortSink.collectedTuples.size());
- for (Object o : sortSink.collectedTuples) {
- LOG.debug(o.toString());
- KeyValPair<TimeBucketKey, Map<ResourceType, Double>> keyValPair = (KeyValPair<TimeBucketKey, Map<ResourceType, Double>>)o;
- Assert.assertEquals("emitted value for 'cpu' was ", 2.0, keyValPair.getValue().get(ResourceType.CPU), 0);
- Assert.assertEquals("emitted value for 'hdd' was ", 1.0, keyValPair.getValue().get(ResourceType.HDD), 0);
- Assert.assertEquals("emitted value for 'ram' was ", 1.0, keyValPair.getValue().get(ResourceType.RAM), 0);
-
- }
- LOG.debug("Done percentile testing\n");
-
- }
-
- public void testStandarDeviation(CalculatorOperator oper)
- {
- CollectorTestSink sortSink = new CollectorTestSink();
- oper.sdOutputPort.setSink(sortSink);
- Calendar calendar = Calendar.getInstance();
- Date date = calendar.getTime();
- String timeKey = minuteDateFormat.format(date);
- String day = calendar.get(Calendar.DAY_OF_MONTH) + "";
-
- Integer vs = new Integer(1);
- MachineKey mk = new MachineKey(timeKey, day, vs, vs, vs, vs, vs, vs, vs);
-
- oper.beginWindow(0);
-
- MachineInfo info = new MachineInfo(mk, 1, 1, 1);
- oper.dataPort.process(info);
-
- info.setCpu(2);
- oper.dataPort.process(info);
-
- info.setCpu(3);
- oper.dataPort.process(info);
-
- oper.endWindow();
-
- Assert.assertEquals("number emitted tuples", 1, sortSink.collectedTuples.size());
- for (Object o : sortSink.collectedTuples) {
- LOG.debug(o.toString());
- KeyValPair<TimeBucketKey, Map<ResourceType, Double>> keyValPair = (KeyValPair<TimeBucketKey, Map<ResourceType, Double>>)o;
- Assert.assertEquals("emitted value for 'cpu' was ", getSD(ImmutableList.of(1, 2, 3)), keyValPair.getValue().get(ResourceType.CPU), 0);
- Assert.assertEquals("emitted value for 'hdd' was ", getSD(ImmutableList.of(1, 1, 1)), keyValPair.getValue().get(ResourceType.HDD), 0);
- Assert.assertEquals("emitted value for 'ram' was ", getSD(ImmutableList.of(1, 1, 1)), keyValPair.getValue().get(ResourceType.RAM), 0);
-
- }
- LOG.debug("Done sd testing\n");
-
- }
-
- private final double getSD(List<Integer> input)
- {
- int sum = 0;
- for (int i : input) {
- sum += i;
- }
- double avg = sum / (input.size() * 1.0);
- double sd = 0;
- for (Integer point : input) {
- sd += Math.pow(point - avg, 2);
- }
- return Math.sqrt(sd);
- }
-
- public void testMax(CalculatorOperator oper)
- {
- CollectorTestSink sortSink = new CollectorTestSink();
- oper.maxOutputPort.setSink(sortSink);
- Calendar calendar = Calendar.getInstance();
- Date date = calendar.getTime();
- String timeKey = minuteDateFormat.format(date);
- String day = calendar.get(Calendar.DAY_OF_MONTH) + "";
-
- Integer vs = new Integer(1);
- MachineKey mk = new MachineKey(timeKey, day, vs, vs, vs, vs, vs, vs, vs);
-
- oper.beginWindow(0);
-
- MachineInfo info = new MachineInfo(mk, 1, 1, 1);
- oper.dataPort.process(info);
-
- info.setCpu(2);
- oper.dataPort.process(info);
-
- info.setCpu(3);
- oper.dataPort.process(info);
-
- oper.endWindow();
-
- Assert.assertEquals("number emitted tuples", 1, sortSink.collectedTuples.size());
- for (Object o : sortSink.collectedTuples) {
- LOG.debug(o.toString());
- KeyValPair<TimeBucketKey, Map<ResourceType, Double>> keyValPair = (KeyValPair<TimeBucketKey, Map<ResourceType, Double>>)o;
- Assert.assertEquals("emitted value for 'cpu' was ", 3, keyValPair.getValue().get(ResourceType.CPU), 0);
- Assert.assertEquals("emitted value for 'hdd' was ", 1, keyValPair.getValue().get(ResourceType.HDD), 0);
- Assert.assertEquals("emitted value for 'ram' was ", 1, keyValPair.getValue().get(ResourceType.RAM), 0);
-
- }
- LOG.debug("Done max testing\n");
-
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/machinedata/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/demos/machinedata/src/test/resources/log4j.properties b/demos/machinedata/src/test/resources/log4j.properties
deleted file mode 100644
index cf0d19e..0000000
--- a/demos/machinedata/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,43 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-log4j.rootLogger=DEBUG,CONSOLE
-
-log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
-log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
-log4j.appender.CONSOLE.threshold=${test.log.console.threshold}
-test.log.console.threshold=DEBUG
-
-log4j.appender.RFA=org.apache.log4j.RollingFileAppender
-log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
-log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
-log4j.appender.RFA.File=/tmp/app.log
-
-# to enable, add SYSLOG to rootLogger
-log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender
-log4j.appender.SYSLOG.syslogHost=127.0.0.1
-log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout
-log4j.appender.SYSLOG.layout.conversionPattern=${dt.cid} %-5p [%t] %c{2} %x - %m%n
-log4j.appender.SYSLOG.Facility=LOCAL1
-
-log4j.logger.org=info
-#log4j.logger.org.apache.commons.beanutils=warn
-log4j.logger.com.datatorrent=debug
-log4j.logger.org.apache.apex=debug
[13/30] apex-malhar git commit: Renamed demos to examples. Packages
and artifactid names are changed as suggested.
Posted by th...@apache.org.
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/MerchantTransactionBucketOperator.java
----------------------------------------------------------------------
diff --git a/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/MerchantTransactionBucketOperator.java b/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/MerchantTransactionBucketOperator.java
new file mode 100644
index 0000000..4a57207
--- /dev/null
+++ b/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/MerchantTransactionBucketOperator.java
@@ -0,0 +1,148 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.frauddetect;
+
+import java.text.DecimalFormat;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.lang.mutable.MutableDouble;
+import org.apache.commons.lang.mutable.MutableLong;
+
+import com.datatorrent.api.DefaultInputPort;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.common.util.BaseOperator;
+import com.datatorrent.lib.util.KeyValPair;
+
+/**
+ * A bucket-like operator to accept merchant transaction object and dissipate the
+ * transaction amount to the further downstream operator for calculating min, max and std-deviation.
+ *
+ * @since 0.9.0
+ */
+public class MerchantTransactionBucketOperator extends BaseOperator
+{
+ private static final Logger logger = LoggerFactory.getLogger(MerchantTransactionBucketOperator.class);
+ /*
+ public final transient DefaultOutputPort<KeyValPair<MerchantKey, String>> binOutputPort =
+ new DefaultOutputPort<KeyValPair<MerchantKey, String>>();
+ */
+ public final transient DefaultOutputPort<KeyValPair<KeyValPair<MerchantKey, String>, Integer>> binCountOutputPort =
+ new DefaultOutputPort<KeyValPair<KeyValPair<MerchantKey, String>, Integer>>();
+ public final transient DefaultOutputPort<KeyValPair<MerchantKey, Long>> txOutputPort =
+ new DefaultOutputPort<KeyValPair<MerchantKey, Long>>();
+ public final transient DefaultOutputPort<KeyValPair<MerchantKey, CreditCardData>> ccAlertOutputPort =
+ new DefaultOutputPort<KeyValPair<MerchantKey, CreditCardData>>();
+ public final transient DefaultOutputPort<Map<String, Object>> summaryTxnOutputPort =
+ new DefaultOutputPort<Map<String, Object>>();
+ private MutableLong totalTxns = new MutableLong(0);
+ private MutableLong txnsInLastSecond = new MutableLong(0);
+ private MutableDouble amtInLastSecond = new MutableDouble(0);
+ private transient DecimalFormat amtFormatter = new DecimalFormat("#.##");
+ public transient DefaultInputPort<MerchantTransaction> inputPort = new DefaultInputPort<MerchantTransaction>()
+ {
+ @Override
+ public void process(MerchantTransaction tuple)
+ {
+ processTuple(tuple);
+ }
+
+ };
+ public transient DefaultInputPort<MerchantTransaction> txUserInputPort = new DefaultInputPort<MerchantTransaction>()
+ {
+ @Override
+ public void process(MerchantTransaction tuple)
+ {
+ processTuple(tuple);
+ }
+
+ };
+
+ public void endWindow()
+ {
+ Map<String, Object> summary = new HashMap<String, Object>();
+ double avg;
+ if (txnsInLastSecond.longValue() == 0) {
+ avg = 0;
+ } else {
+ avg = amtInLastSecond.doubleValue() / txnsInLastSecond.longValue();
+ }
+ summary.put("totalTxns", totalTxns);
+ summary.put("txnsInLastSecond", txnsInLastSecond);
+ summary.put("amtInLastSecond", amtFormatter.format(amtInLastSecond));
+ summary.put("avgAmtInLastSecond", amtFormatter.format(avg));
+ summaryTxnOutputPort.emit(summary);
+ txnsInLastSecond.setValue(0);
+ amtInLastSecond.setValue(0);
+ }
+
+ private void processTuple(MerchantTransaction tuple)
+ {
+ emitBankIdNumTuple(tuple, binCountOutputPort);
+ emitMerchantKeyTuple(tuple, txOutputPort);
+ emitCreditCardKeyTuple(tuple, ccAlertOutputPort);
+ totalTxns.increment();
+ txnsInLastSecond.increment();
+ amtInLastSecond.add(tuple.amount);
+ }
+
+ private void emitMerchantKeyTuple(MerchantTransaction tuple, DefaultOutputPort<KeyValPair<MerchantKey, Long>> outputPort)
+ {
+ MerchantKey key = getMerchantKey(tuple);
+ KeyValPair<MerchantKey, Long> keyValPair = new KeyValPair<MerchantKey, Long>(key, tuple.amount);
+ outputPort.emit(keyValPair);
+ }
+
+ //private void emitBankIdNumTuple(MerchantTransaction tuple, DefaultOutputPort<KeyValPair<MerchantKey, String>> outputPort)
+ private void emitBankIdNumTuple(MerchantTransaction tuple, DefaultOutputPort<KeyValPair<KeyValPair<MerchantKey, String>, Integer>> outputPort)
+ {
+ MerchantKey key = getMerchantKey(tuple);
+ KeyValPair<MerchantKey, String> keyValPair = new KeyValPair<MerchantKey, String>(key, tuple.bankIdNum);
+ outputPort.emit(new KeyValPair<KeyValPair<MerchantKey, String>, Integer>(keyValPair, 1));
+ }
+
+ private void emitCreditCardKeyTuple(MerchantTransaction tuple, DefaultOutputPort<KeyValPair<MerchantKey, CreditCardData>> outputPort)
+ {
+ MerchantKey key = getMerchantKey(tuple);
+
+ CreditCardData data = new CreditCardData();
+ data.fullCcNum = tuple.fullCcNum;
+ data.amount = tuple.amount;
+
+ KeyValPair<MerchantKey, CreditCardData> keyValPair = new KeyValPair<MerchantKey, CreditCardData>(key, data);
+ outputPort.emit(keyValPair);
+ }
+
+ private MerchantKey getMerchantKey(MerchantTransaction tuple)
+ {
+ MerchantKey key = new MerchantKey();
+ key.merchantId = tuple.merchantId;
+ key.terminalId = tuple.terminalId;
+ key.zipCode = tuple.zipCode;
+ key.country = tuple.country;
+ key.merchantType = tuple.merchantType;
+ key.userGenerated = tuple.userGenerated;
+ key.time = tuple.time;
+ return key;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/MerchantTransactionGenerator.java
----------------------------------------------------------------------
diff --git a/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/MerchantTransactionGenerator.java b/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/MerchantTransactionGenerator.java
new file mode 100644
index 0000000..2327344
--- /dev/null
+++ b/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/MerchantTransactionGenerator.java
@@ -0,0 +1,210 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.frauddetect;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.apex.examples.frauddetect.util.JsonUtils;
+
+import com.datatorrent.api.Context;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.InputOperator;
+
+import com.datatorrent.common.util.BaseOperator;
+
+/**
+ * Information tuple generator with randomness.
+ *
+ * @since 0.9.0
+ */
+public class MerchantTransactionGenerator extends BaseOperator implements InputOperator
+{
+ private final Random randomNum = new Random();
+ public static final int[] zipCodes = {94086, 94087, 94088, 94089, 94090, 94091, 94092, 94093};
+ public static final String[] merchantIds = {"Wal-Mart", "Target", "Amazon", "Apple", "Sears", "Macys", "JCPenny", "Levis"};
+// public static final String bankIdNums[] = { "1111 1111 1111", "2222 2222 2222", "3333 3333 3333", "4444 4444 4444", "5555 5555 5555", "6666 6666 6666", "7777 7777 7777", "8888 8888 8888"};
+// public static final String ccNums[] = { "0001", "0002", "0003", "0004", "0005", "0006", "0007", "0008"};
+// public static final String bankIdNums[] = { "1111 1111 1111", "2222 2222 2222", "3333 3333 3333", "4444 4444 4444"};
+// public static final String ccNums[] = { "0001", "0002", "0003", "0004"};
+// public static final int zipCodes[] = { 94086, 94087, 94088, 94089, 94090};
+// public static final String merchantIds[] = { "Wal-Mart", "Target", "Amazon", "Apple"};
+// private int bankIdNumMin = 0;
+// private int bankIdNumMax = bankIdNums.length - 1;
+// private int ccMin = 0;
+// private int ccMax = ccNums.length - 1;
+ private int amountMin = 1;
+ private int amountMax = 400;
+ private int merchantIdMin = 0;
+ private int merchantIdMax = merchantIds.length - 1;
+ private int terminalIdMin = 1;
+ private int terminalIdMax = 8;
+ private int zipMin = 0;
+ private int zipMax = zipCodes.length - 1;
+ private int tupleBlastSize = 2000;
+ private boolean stopGeneration = false;
+
+ @Override
+ public void setup(Context.OperatorContext context)
+ {
+ super.setup(context);
+ }
+
+ public transient DefaultOutputPort<MerchantTransaction> txOutputPort =
+ new DefaultOutputPort<MerchantTransaction>();
+ public transient DefaultOutputPort<String> txDataOutputPort =
+ new DefaultOutputPort<String>();
+
+ @Override
+ public void emitTuples()
+ {
+ int count = 0;
+ List<MerchantTransaction> txList = new ArrayList();
+
+ while (!stopGeneration && count < getTupleBlastSize()) {
+
+ String bankIdNum = genBankIdNum();
+ String ccNum = genCC();
+ int merchant = genMerchantId();
+ int terminal = genTerminalId();
+ int zip = genZip();
+
+ long amount = genAmount();
+
+// int bankIdNum = 1;
+// int ccNum = 2;
+// long amount = 5000;
+// int merchant = 3;
+// int terminal = 4;
+// int zip = 0;
+
+ MerchantTransaction tx = new MerchantTransaction();
+ tx.bankIdNum = bankIdNum;
+ tx.ccNum = ccNum;
+ tx.fullCcNum = tx.bankIdNum + " " + tx.ccNum;
+ tx.amount = amount;
+ tx.merchantId = merchantIds[merchant];
+
+ // its INTERNET merchant
+ tx.merchantType = merchant == 2 || merchant == 3
+ ? MerchantTransaction.MerchantType.INTERNET
+ : MerchantTransaction.MerchantType.BRICK_AND_MORTAR;
+
+ tx.transactionType = MerchantTransaction.TransactionType.POS;
+
+ // set terminal only for a BRICK_AND_MORTAR merchant
+ if (merchant != 2 && merchant != 3) {
+ tx.terminalId = terminal;
+ }
+ tx.zipCode = zipCodes[zip];
+ tx.country = "USA";
+ tx.time = System.currentTimeMillis();
+
+ tx.userGenerated = false;
+
+ txOutputPort.emit(tx);
+
+ txList.add(tx);
+
+ count++;
+ }
+ for (MerchantTransaction txData : txList) {
+ try {
+ txDataOutputPort.emit(JsonUtils.toJson(txData));
+ } catch (IOException e) {
+ logger.warn("Exception while converting object to JSON", e);
+ }
+ }
+ txList.clear();
+
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+
+ }
+
+ public String genBankIdNum()
+ {
+ // Bank ID will be between 1000 0000 and 3500 0000 (25 BINs)
+ int base = randomNum.nextInt(100) + 100;
+ return base + "0 0000";
+ }
+
+ public String genCC()
+ {
+ // CC will be 1000 0000 to 1400 0000 (400,000 cards per BIN)
+ int base = (randomNum.nextInt(100000) + 10000000);
+ String baseString = Integer.toString(base);
+ return baseString.substring(0, 4) + " " + baseString.substring(4);
+ }
+
+ public int genAmount()
+ {
+ int lowRange = 50;
+ int range = amountMax - amountMin + randomNum.nextInt(lowRange);
+ return amountMin + randomNum.nextInt(range);
+ }
+
+ public int genMerchantId()
+ {
+ int range = merchantIdMax - merchantIdMin + 1;
+ return merchantIdMin + randomNum.nextInt(range);
+ }
+
+ public int genTerminalId()
+ {
+ int range = terminalIdMax - terminalIdMin + 1;
+ return terminalIdMin + randomNum.nextInt(range);
+ }
+
+ public int genZip()
+ {
+ int range = zipMax - zipMin + 1;
+ return zipMin + randomNum.nextInt(range);
+ }
+
+ public void setStopGeneration(boolean stopGeneration)
+ {
+ this.stopGeneration = stopGeneration;
+ }
+
+ /**
+ * @return the tupleBlastSize
+ */
+ public int getTupleBlastSize()
+ {
+ return tupleBlastSize;
+ }
+
+ /**
+ * @param tupleBlastSize the tupleBlastSize to set
+ */
+ public void setTupleBlastSize(int tupleBlastSize)
+ {
+ this.tupleBlastSize = tupleBlastSize;
+ }
+
+ private static final Logger logger = LoggerFactory.getLogger(MerchantTransactionGenerator.class);
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/MerchantTransactionInputHandler.java
----------------------------------------------------------------------
diff --git a/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/MerchantTransactionInputHandler.java b/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/MerchantTransactionInputHandler.java
new file mode 100644
index 0000000..6dd9c2f
--- /dev/null
+++ b/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/MerchantTransactionInputHandler.java
@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.frauddetect;
+
+import java.util.Map;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.datatorrent.api.DefaultInputPort;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.common.util.BaseOperator;
+
+/**
+ * Common utility class that can be used by all other operators to handle user input
+ * captured from the Web socket input port.
+ *
+ * @since 0.9.0
+ */
+public class MerchantTransactionInputHandler extends BaseOperator
+{
+ public static final String KEY_BANK_ID_NUMBER = "bankIdNum"; // first 12 digits
+ public static final String KEY_CREDIT_CARD_NUMBER = "ccNum"; // last 4 digits
+ public static final String KEY_MERCHANT_ID = "merchantId";
+ public static final String KEY_TERMINAL_ID = "terminalId";
+ public static final String KEY_ZIP_CODE = "zipCode";
+ public static final String KEY_AMOUNT = "amount";
+ public transient DefaultOutputPort<MerchantTransaction> txOutputPort =
+ new DefaultOutputPort<MerchantTransaction>();
+ public transient DefaultInputPort<Map<String, String>> userTxInputPort = new DefaultInputPort<Map<String, String>>()
+ {
+ @Override
+ public void process(Map<String, String> tuple)
+ {
+ try {
+ txOutputPort.emit(processInput(tuple));
+ } catch (Exception exc) {
+ logger.error("Exception while handling the input", exc);
+ }
+ }
+
+ };
+
+ public MerchantTransaction processInput(Map<String, String> tuple)
+ {
+ String bankIdNum = null;
+ String ccNum = null;
+ String merchantId = null;
+ Integer terminalId = null;
+ Integer zipCode = null;
+ Long amount = null;
+ for (Map.Entry<String, String> e : tuple.entrySet()) {
+ if (e.getKey().equals(KEY_BANK_ID_NUMBER)) {
+ bankIdNum = e.getValue();
+ }
+ if (e.getKey().equals(KEY_CREDIT_CARD_NUMBER)) {
+ ccNum = e.getValue();
+ }
+ if (e.getKey().equals(KEY_MERCHANT_ID)) {
+ merchantId = e.getValue();
+ }
+ if (e.getKey().equals(KEY_TERMINAL_ID)) {
+ terminalId = new Integer(e.getValue());
+ }
+ if (e.getKey().equals(KEY_ZIP_CODE)) {
+ zipCode = new Integer(e.getValue());
+ }
+ if (e.getKey().equals(KEY_AMOUNT)) {
+ amount = new Long(e.getValue());
+ }
+ }
+
+ if (bankIdNum == null || ccNum == null || merchantId == null || terminalId == null || zipCode == null || amount == null) {
+ throw new IllegalArgumentException("Missing required input!");
+ }
+
+ MerchantTransaction tx = new MerchantTransaction();
+ tx.bankIdNum = bankIdNum;
+ tx.ccNum = ccNum;
+ tx.fullCcNum = bankIdNum + " " + ccNum;
+ tx.merchantId = merchantId;
+ tx.terminalId = terminalId;
+ tx.zipCode = zipCode;
+ tx.country = "USA";
+ tx.amount = amount;
+ tx.merchantType = tx.merchantId.equalsIgnoreCase(MerchantTransactionGenerator.merchantIds[2])
+ || tx.merchantId.equalsIgnoreCase(MerchantTransactionGenerator.merchantIds[3])
+ ? MerchantTransaction.MerchantType.INTERNET
+ : MerchantTransaction.MerchantType.BRICK_AND_MORTAR;
+ tx.transactionType = MerchantTransaction.TransactionType.POS;
+
+ tx.userGenerated = true;
+ tx.time = System.currentTimeMillis();
+
+ return tx;
+
+ }
+
+ private static final Logger logger = LoggerFactory.getLogger(MerchantTransactionInputHandler.class);
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/SlidingWindowSumKeyVal.java
----------------------------------------------------------------------
diff --git a/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/SlidingWindowSumKeyVal.java b/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/SlidingWindowSumKeyVal.java
new file mode 100644
index 0000000..dc2c942
--- /dev/null
+++ b/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/SlidingWindowSumKeyVal.java
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.frauddetect;
+
+import java.util.ArrayList;
+
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.annotation.OutputPortFieldAnnotation;
+
+import com.datatorrent.lib.multiwindow.AbstractSlidingWindowKeyVal;
+import com.datatorrent.lib.util.KeyValPair;
+
+
+/**
+ * Sliding window sum operator
+ *
+ * @since 0.9.0
+ */
+public class SlidingWindowSumKeyVal<K, V extends Number> extends AbstractSlidingWindowKeyVal<K, V, SlidingWindowSumObject>
+{
+
+ /**
+ * Output port to emit simple moving average (SMA) of last N window as Double.
+ */
+ @OutputPortFieldAnnotation(optional = true)
+ public final transient DefaultOutputPort<KeyValPair<K, Double>> doubleSum = new DefaultOutputPort<KeyValPair<K, Double>>();
+ /**
+ * Output port to emit simple moving average (SMA) of last N window as Float.
+ */
+ @OutputPortFieldAnnotation(optional = true)
+ public final transient DefaultOutputPort<KeyValPair<K, Float>> floatSum = new DefaultOutputPort<KeyValPair<K, Float>>();
+ /**
+ * Output port to emit simple moving average (SMA) of last N window as Long.
+ */
+ @OutputPortFieldAnnotation(optional = true)
+ public final transient DefaultOutputPort<KeyValPair<K, Long>> longSum = new DefaultOutputPort<KeyValPair<K, Long>>();
+ /**
+ * Output port to emit simple moving average (SMA) of last N window as
+ * Integer.
+ */
+ @OutputPortFieldAnnotation(optional = true)
+ public final transient DefaultOutputPort<KeyValPair<K, Integer>> integerSum = new DefaultOutputPort<KeyValPair<K, Integer>>();
+
+
+ @Override
+ public void processDataTuple(KeyValPair<K, V> tuple)
+ {
+ K key = tuple.getKey();
+ ArrayList<SlidingWindowSumObject> stateList = buffer.get(key);
+ if (stateList == null) {
+ stateList = new ArrayList<SlidingWindowSumObject>();
+ for (int i = 0; i < windowSize; ++i) {
+ stateList.add(new SlidingWindowSumObject());
+ }
+ buffer.put(key, stateList);
+ }
+ SlidingWindowSumObject state = stateList.get(currentstate);
+ state.add(tuple.getValue());
+ }
+
+ @Override
+ public void emitTuple(K key, ArrayList<SlidingWindowSumObject> obj)
+ {
+ double sum = 0;
+ for (int i = 0; i < obj.size(); ++i) {
+ SlidingWindowSumObject state = obj.get(i);
+ sum += state.getSum();
+ }
+ if (doubleSum.isConnected()) {
+ doubleSum.emit(new KeyValPair<K, Double>(key, sum));
+ }
+ if (floatSum.isConnected()) {
+ floatSum.emit(new KeyValPair<K, Float>(key, (float)sum));
+ }
+ if (longSum.isConnected()) {
+ longSum.emit(new KeyValPair<K, Long>(key, (long)sum));
+ }
+ if (integerSum.isConnected()) {
+ integerSum.emit(new KeyValPair<K, Integer>(key, (int)sum));
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/SlidingWindowSumObject.java
----------------------------------------------------------------------
diff --git a/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/SlidingWindowSumObject.java b/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/SlidingWindowSumObject.java
new file mode 100644
index 0000000..fc5f95d
--- /dev/null
+++ b/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/SlidingWindowSumObject.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.frauddetect;
+
+
+import org.apache.commons.lang.mutable.MutableDouble;
+
+import com.datatorrent.lib.multiwindow.SimpleMovingAverageObject;
+
+/**
+ * State object for sliding window sum
+ *
+ * @since 0.9.0
+ */
+public class SlidingWindowSumObject extends SimpleMovingAverageObject
+{
+
+ MutableDouble sum = new MutableDouble(0);
+
+ public void add(Number n)
+ {
+ sum.add(n);
+ }
+
+ @Override
+ public double getSum()
+ {
+ return sum.doubleValue();
+ }
+
+ @Override
+ public void clear()
+ {
+ sum.setValue(0);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/TransactionStatsAggregator.java
----------------------------------------------------------------------
diff --git a/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/TransactionStatsAggregator.java b/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/TransactionStatsAggregator.java
new file mode 100644
index 0000000..71f3035
--- /dev/null
+++ b/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/TransactionStatsAggregator.java
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.frauddetect;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.apex.examples.frauddetect.util.JsonUtils;
+
+import com.datatorrent.api.DefaultInputPort;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.common.util.BaseOperator;
+import com.datatorrent.lib.util.HighLow;
+import com.datatorrent.lib.util.KeyValPair;
+
+/**
+ * Operator to aggregate the min, max, sma, std-dev and variance for the given key.
+ *
+ * @since 0.9.0
+ */
+public class TransactionStatsAggregator extends BaseOperator
+{
+ public Map<MerchantKey, TransactionStatsData> aggrgateMap =
+ new HashMap<MerchantKey, TransactionStatsData>();
+ public final transient DefaultOutputPort<String> txDataOutputPort = new DefaultOutputPort<String>();
+ public final transient DefaultInputPort<KeyValPair<MerchantKey, HighLow<Long>>> rangeInputPort =
+ new DefaultInputPort<KeyValPair<MerchantKey, HighLow<Long>>>()
+ {
+ @Override
+ public void process(KeyValPair<MerchantKey, HighLow<Long>> tuple)
+ {
+ TransactionStatsData data = getDataObjectFromMap(tuple.getKey());
+ // HighLow is not currently typed, casting till it is fixed
+ data.min = tuple.getValue().getLow();
+ data.max = tuple.getValue().getHigh();
+ }
+
+ };
+ public final transient DefaultInputPort<KeyValPair<MerchantKey, Long>> smaInputPort =
+ new DefaultInputPort<KeyValPair<MerchantKey, Long>>()
+ {
+ @Override
+ public void process(KeyValPair<MerchantKey, Long> tuple)
+ {
+ TransactionStatsData data = getDataObjectFromMap(tuple.getKey());
+ data.sma = tuple.getValue();
+ }
+
+ };
+
+ private TransactionStatsData getDataObjectFromMap(MerchantKey key)
+ {
+ TransactionStatsData data = aggrgateMap.get(key);
+ if (data == null) {
+ data = new TransactionStatsData();
+ data.time = System.currentTimeMillis();
+ data.merchantId = key.merchantId;
+ data.terminalId = key.terminalId == null ? 0 : key.terminalId;
+ data.zipCode = key.zipCode;
+ data.merchantType = key.merchantType;
+ aggrgateMap.put(key, data);
+ }
+ return data;
+ }
+
+ @Override
+ public void endWindow()
+ {
+ for (Map.Entry<MerchantKey, TransactionStatsData> entry : aggrgateMap.entrySet()) {
+ try {
+ txDataOutputPort.emit(JsonUtils.toJson(entry.getValue()));
+ } catch (IOException e) {
+ logger.warn("Exception while converting object to JSON", e);
+ }
+ }
+ aggrgateMap.clear();
+ }
+
+ private static final Logger logger = LoggerFactory.getLogger(TransactionStatsAggregator.class);
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/TransactionStatsData.java
----------------------------------------------------------------------
diff --git a/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/TransactionStatsData.java b/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/TransactionStatsData.java
new file mode 100644
index 0000000..f0b2b86
--- /dev/null
+++ b/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/TransactionStatsData.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.frauddetect;
+
+/**
+ * POJO to capture transaction data related to min, max, sma, std-dev, variance.
+ *
+ * @since 0.9.0
+ */
+public class TransactionStatsData
+{
+ public String merchantId;
+ public int terminalId;
+ public int zipCode;
+ public MerchantTransaction.MerchantType merchantType;
+ public long min;
+ public long max;
+ public double sma;
+ public long time;
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/operator/HdfsStringOutputOperator.java
----------------------------------------------------------------------
diff --git a/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/operator/HdfsStringOutputOperator.java b/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/operator/HdfsStringOutputOperator.java
new file mode 100644
index 0000000..89c4bcd
--- /dev/null
+++ b/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/operator/HdfsStringOutputOperator.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.frauddetect.operator;
+
+import java.io.File;
+
+import com.datatorrent.api.Context.DAGContext;
+import com.datatorrent.api.Context.OperatorContext;
+import com.datatorrent.lib.io.fs.AbstractFileOutputOperator;
+
+/**
+ * Adapter for writing Strings to HDFS
+ * <p>
+ * Serializes tuples into a HDFS file.<br/>
+ * </p>
+ *
+ * @since 0.9.4
+ */
+public class HdfsStringOutputOperator extends AbstractFileOutputOperator<String>
+{
+ private transient String outputFileName;
+ private transient String contextId;
+ private int index = 0;
+
+ public HdfsStringOutputOperator()
+ {
+ setMaxLength(1024 * 1024);
+ }
+
+ @Override
+ public void setup(OperatorContext context)
+ {
+ contextId = context.getValue(DAGContext.APPLICATION_NAME);
+ outputFileName = File.separator + contextId +
+ File.separator + "transactions.out.part";
+ super.setup(context);
+ }
+
+ @Override
+ public byte[] getBytesForTuple(String t)
+ {
+ return t.getBytes();
+ }
+
+ @Override
+ protected String getFileName(String tuple)
+ {
+ return outputFileName;
+ }
+
+ @Override
+ public String getPartFileName(String fileName, int part)
+ {
+ return fileName + part;
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/operator/MongoDBOutputOperator.java
----------------------------------------------------------------------
diff --git a/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/operator/MongoDBOutputOperator.java b/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/operator/MongoDBOutputOperator.java
new file mode 100644
index 0000000..e059c03
--- /dev/null
+++ b/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/operator/MongoDBOutputOperator.java
@@ -0,0 +1,188 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.frauddetect.operator;
+
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.validation.constraints.NotNull;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.mongodb.DB;
+import com.mongodb.DBCollection;
+import com.mongodb.DBObject;
+import com.mongodb.MongoClient;
+import com.mongodb.WriteConcern;
+import com.mongodb.WriteResult;
+import com.mongodb.util.JSON;
+
+import com.datatorrent.api.Context;
+import com.datatorrent.api.DefaultInputPort;
+import com.datatorrent.common.util.BaseOperator;
+
+
+/**
+ * Operator to write data into MongoDB
+ *
+ * @since 0.9.0
+ */
+public class MongoDBOutputOperator extends BaseOperator
+{
+ @NotNull
+ protected String hostName;
+ @NotNull
+ protected String dataBase;
+ @NotNull
+ protected String collection;
+
+ protected WriteConcern writeConcern = WriteConcern.ACKNOWLEDGED;
+
+ protected String userName;
+ protected String passWord;
+
+ protected transient MongoClient mongoClient;
+ protected transient DB db;
+ protected transient DBCollection dbCollection;
+
+ protected List<DBObject> dataList = new ArrayList<DBObject>();
+
+ public MongoDBOutputOperator()
+ {
+ }
+
+ /**
+ * Take the JSON formatted string and convert it to DBObject
+ */
+ public final transient DefaultInputPort<String> inputPort = new DefaultInputPort<String>()
+ {
+ @Override
+ public void process(String tuple)
+ {
+ dataList.add((DBObject)JSON.parse(tuple));
+ }
+ };
+
+ @Override
+ public void setup(Context.OperatorContext context)
+ {
+ super.setup(context);
+ try {
+ mongoClient = new MongoClient(hostName);
+ db = mongoClient.getDB(dataBase);
+ if (userName != null && passWord != null) {
+ if (!db.authenticate(userName, passWord.toCharArray())) {
+ throw new IllegalArgumentException("MongoDB authentication failed. Illegal username and password for MongoDB!!");
+ }
+ }
+ dbCollection = db.getCollection(collection);
+ } catch (UnknownHostException ex) {
+ logger.debug(ex.toString());
+ }
+ }
+
+ @Override
+ public void beginWindow(long windowId)
+ {
+ // nothing
+ }
+
+ @Override
+ public void endWindow()
+ {
+ logger.debug("mongo datalist size: " + dataList.size());
+ if (dataList.size() > 0) {
+ WriteResult result = dbCollection.insert(dataList, writeConcern);
+ logger.debug("Result for MongoDB insert: " + result);
+ dataList.clear();
+ }
+ }
+
+ @Override
+ public void teardown()
+ {
+ if (mongoClient != null) {
+ mongoClient.close();
+ }
+ }
+
+ public String getHostName()
+ {
+ return hostName;
+ }
+
+ public void setHostName(String hostName)
+ {
+ this.hostName = hostName;
+ }
+
+ public String getDataBase()
+ {
+ return dataBase;
+ }
+
+ public void setDataBase(String dataBase)
+ {
+ this.dataBase = dataBase;
+ }
+
+ public String getCollection()
+ {
+ return collection;
+ }
+
+ public void setCollection(String collection)
+ {
+ this.collection = collection;
+ }
+
+ public String getUserName()
+ {
+ return userName;
+ }
+
+ public void setUserName(String userName)
+ {
+ this.userName = userName;
+ }
+
+ public String getPassWord()
+ {
+ return passWord;
+ }
+
+ public void setPassWord(String passWord)
+ {
+ this.passWord = passWord;
+ }
+
+ public WriteConcern getWriteConcern()
+ {
+ return writeConcern;
+ }
+
+ public void setWriteConcern(WriteConcern writeConcern)
+ {
+ this.writeConcern = writeConcern;
+ }
+
+ private static final Logger logger = LoggerFactory.getLogger(MongoDBOutputOperator.class);
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/util/JsonUtils.java
----------------------------------------------------------------------
diff --git a/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/util/JsonUtils.java b/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/util/JsonUtils.java
new file mode 100644
index 0000000..932ef64
--- /dev/null
+++ b/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/util/JsonUtils.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.frauddetect.util;
+
+import java.io.IOException;
+
+import org.codehaus.jackson.map.ObjectMapper;
+
+/**
+ * Utility class to deal with JSON and Object
+ *
+ * @since 0.9.0
+ */
+public class JsonUtils
+{
+ private static final ObjectMapper mapper = new ObjectMapper();
+
+ public static String toJson(Object obj) throws IOException
+ {
+ return mapper.writeValueAsString(obj);
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/frauddetect/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/examples/frauddetect/src/main/resources/META-INF/properties.xml b/examples/frauddetect/src/main/resources/META-INF/properties.xml
new file mode 100644
index 0000000..7a42ac4
--- /dev/null
+++ b/examples/frauddetect/src/main/resources/META-INF/properties.xml
@@ -0,0 +1,167 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<configuration>
+<property>
+ <name>dt.attr.STREAMING_WINDOW_SIZE_MILLIS</name>
+ <value>1000</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.ccUserAlertQueryOutput.topic</name>
+ <value>examples.app.frauddetect.fraudAlert</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.binUserAlertOutput.topic</name>
+ <value>examples.app.frauddetect.fraudAlert</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.avgUserAlertQueryOutput.topic</name>
+ <value>examples.app.frauddetect.fraudAlert</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.txSummaryWsOutput.topic</name>
+ <value>examples.app.frauddetect.txSummary</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.bankInfoFraudDetector.attr.APPLICATION_WINDOW_COUNT
+ </name>
+ <value>10</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.bankInfoFraudDetector.threshold
+ </name>
+ <value>20</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.rangePerMerchant.attr.APPLICATION_WINDOW_COUNT
+ </name>
+ <value>1</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.txReceiver.attr.APPLICATION_WINDOW_COUNT
+ </name>
+ <value>1</value>
+</property>
+<!-- property>
+ <name>dt.application.frauddetect.operator.smaPerMerchant.attr.APPLICATION_WINDOW_COUNT
+ </name>
+ <value>1</value>
+</property-->
+<property>
+ <name>dt.application.FraudDetectExample.operator.smaPerMerchant.windowSize
+ </name>
+ <value>30</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.movingSum.attr.APPLICATION_WINDOW_COUNT
+ </name>
+ <value>10</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.movingSum.windowSize
+ </name>
+ <value>3</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.avgAlerter.attr.APPLICATION_WINDOW_COUNT
+ </name>
+ <value>1</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.avgAlerter.threshold
+ </name>
+ <value>1200</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.amountFraudDetector.attr.APPLICATION_WINDOW_COUNT
+ </name>
+ <value>1</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.amountFraudDetector.threshold
+ </name>
+ <value>420</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.mongoTxStatsOutput.hostName</name>
+ <value>localhost</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.mongoTxStatsOutput.dataBase</name>
+ <value>frauddetect</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.mongoTxStatsOutput.collection</name>
+ <value>txStats</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.mongoBinAlertsOutput.hostName</name>
+ <value>localhost</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.mongoBinAlertsOutput.dataBase</name>
+ <value>frauddetect</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.mongoBinAlertsOutput.collection</name>
+ <value>binAlerts</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.mongoCcAlertsOutput.hostName</name>
+ <value>localhost</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.mongoCcAlertsOutput.dataBase</name>
+ <value>frauddetect</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.mongoCcAlertsOutput.collection</name>
+ <value>ccAlerts</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.mongoAvgAlertsOutput.hostName</name>
+ <value>localhost</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.mongoAvgAlertsOutput.dataBase</name>
+ <value>frauddetect</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.mongoAvgAlertsOutput.collection</name>
+ <value>avgAlerts</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.txStatsAggregator.attr.APPLICATION_WINDOW_COUNT
+ </name>
+ <value>1</value>
+</property>
+
+<property>
+ <name>dt.application.FraudDetectExample.port.*.attr.QUEUE_CAPACITY</name>
+ <value>32000</value>
+ </property>
+ <property>
+ <name>dt.application.FraudDetectExample.operator.*.attr.MEMORY_MB</name>
+ <value>2048</value>
+ </property>
+
+</configuration>
+
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/frauddetect/src/test/java/org/apache/apex/examples/frauddetect/FrauddetectApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/frauddetect/src/test/java/org/apache/apex/examples/frauddetect/FrauddetectApplicationTest.java b/examples/frauddetect/src/test/java/org/apache/apex/examples/frauddetect/FrauddetectApplicationTest.java
new file mode 100644
index 0000000..ffb1cf2
--- /dev/null
+++ b/examples/frauddetect/src/test/java/org/apache/apex/examples/frauddetect/FrauddetectApplicationTest.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.frauddetect;
+
+import org.junit.Test;
+import org.apache.hadoop.conf.Configuration;
+import com.datatorrent.api.LocalMode;
+
+/**
+ * Fraud detection application test
+ */
+public class FrauddetectApplicationTest
+{
+
+ public FrauddetectApplicationTest()
+ {
+ }
+
+ @Test
+ public void testApplication() throws Exception
+ {
+ try {
+ Application application = new Application();
+ Configuration conf = new Configuration(false);
+ conf.addResource("dt-site-frauddetect.xml");
+ LocalMode lma = LocalMode.newInstance();
+ lma.prepareDAG(application, conf);
+ lma.getController().run(120000);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/frauddetect/src/test/resources/dt-site-frauddetect.xml
----------------------------------------------------------------------
diff --git a/examples/frauddetect/src/test/resources/dt-site-frauddetect.xml b/examples/frauddetect/src/test/resources/dt-site-frauddetect.xml
new file mode 100644
index 0000000..7a404c4
--- /dev/null
+++ b/examples/frauddetect/src/test/resources/dt-site-frauddetect.xml
@@ -0,0 +1,173 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<configuration>
+ <property>
+ <name>dt.application.FraudDetectExample.class</name>
+ <value>com.datatorrent.examples.frauddetect.Application</value>
+ <description>An alias for the application</description>
+</property>
+<property>
+ <name>dt.attr.STREAMING_WINDOW_SIZE_MILLIS</name>
+ <value>1000</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.ccUserAlertQueryOutput.topic</name>
+ <value>examples.app.frauddetect.fraudAlert</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.binUserAlertOutput.topic</name>
+ <value>examples.app.frauddetect.fraudAlert</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.avgUserAlertQueryOutput.topic</name>
+ <value>examples.app.frauddetect.fraudAlert</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.txSummaryWsOutput.topic</name>
+ <value>examples.app.frauddetect.txSummary</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.bankInfoFraudDetector.attr.APPLICATION_WINDOW_COUNT
+ </name>
+ <value>10</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.bankInfoFraudDetector.threshold
+ </name>
+ <value>20</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.rangePerMerchant.attr.APPLICATION_WINDOW_COUNT
+ </name>
+ <value>1</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.txReceiver.attr.APPLICATION_WINDOW_COUNT
+ </name>
+ <value>1</value>
+</property>
+<!-- property>
+ <name>dt.application.frauddetect.operator.smaPerMerchant.attr.APPLICATION_WINDOW_COUNT
+ </name>
+ <value>1</value>
+</property-->
+<property>
+ <name>dt.application.FraudDetectExample.operator.smaPerMerchant.windowSize
+ </name>
+ <value>30</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.movingSum.attr.APPLICATION_WINDOW_COUNT
+ </name>
+ <value>10</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.movingSum.windowSize
+ </name>
+ <value>3</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.avgAlerter.attr.APPLICATION_WINDOW_COUNT
+ </name>
+ <value>1</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.avgAlerter.threshold
+ </name>
+ <value>1200</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.amountFraudDetector.attr.APPLICATION_WINDOW_COUNT
+ </name>
+ <value>1</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.amountFraudDetector.threshold
+ </name>
+ <value>420</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.mongoTxStatsOutput.hostName</name>
+ <value>localhost</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.mongoTxStatsOutput.dataBase</name>
+ <value>frauddetect</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.mongoTxStatsOutput.collection</name>
+ <value>txStats</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.mongoBinAlertsOutput.hostName</name>
+ <value>localhost</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.mongoBinAlertsOutput.dataBase</name>
+ <value>frauddetect</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.mongoBinAlertsOutput.collection</name>
+ <value>binAlerts</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.mongoCcAlertsOutput.hostName</name>
+ <value>localhost</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.mongoCcAlertsOutput.dataBase</name>
+ <value>frauddetect</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.mongoCcAlertsOutput.collection</name>
+ <value>ccAlerts</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.mongoAvgAlertsOutput.hostName</name>
+ <value>localhost</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.mongoAvgAlertsOutput.dataBase</name>
+ <value>frauddetect</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.mongoAvgAlertsOutput.collection</name>
+ <value>avgAlerts</value>
+</property>
+<property>
+ <name>dt.application.FraudDetectExample.operator.txStatsAggregator.attr.APPLICATION_WINDOW_COUNT
+ </name>
+ <value>1</value>
+</property>
+
+<property>
+ <name>dt.application.FraudDetectExample.port.*.attr.QUEUE_CAPACITY</name>
+ <value>32000</value>
+ </property>
+<property>
+ <name>dt.application.FraudDetectExampleq.operator.*.attr.MEMORY_MB</name>
+ <value>2048</value>
+</property>
+
+
+</configuration>
+
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/frauddetect/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/examples/frauddetect/src/test/resources/log4j.properties b/examples/frauddetect/src/test/resources/log4j.properties
new file mode 100644
index 0000000..cf0d19e
--- /dev/null
+++ b/examples/frauddetect/src/test/resources/log4j.properties
@@ -0,0 +1,43 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+log4j.rootLogger=DEBUG,CONSOLE
+
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+log4j.appender.CONSOLE.threshold=${test.log.console.threshold}
+test.log.console.threshold=DEBUG
+
+log4j.appender.RFA=org.apache.log4j.RollingFileAppender
+log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
+log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+log4j.appender.RFA.File=/tmp/app.log
+
+# to enable, add SYSLOG to rootLogger
+log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender
+log4j.appender.SYSLOG.syslogHost=127.0.0.1
+log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout
+log4j.appender.SYSLOG.layout.conversionPattern=${dt.cid} %-5p [%t] %c{2} %x - %m%n
+log4j.appender.SYSLOG.Facility=LOCAL1
+
+log4j.logger.org=info
+#log4j.logger.org.apache.commons.beanutils=warn
+log4j.logger.com.datatorrent=debug
+log4j.logger.org.apache.apex=debug
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/highlevelapi/pom.xml
----------------------------------------------------------------------
diff --git a/examples/highlevelapi/pom.xml b/examples/highlevelapi/pom.xml
new file mode 100644
index 0000000..da843b7
--- /dev/null
+++ b/examples/highlevelapi/pom.xml
@@ -0,0 +1,141 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>malhar-examples-highlevelapi</artifactId>
+ <packaging>jar</packaging>
+
+ <name>Apache Apex Malhar High-Level API Example</name>
+ <description>Apex exmaple applications that use High-level API to construct a dag</description>
+
+ <parent>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-examples</artifactId>
+ <version>3.7.0-SNAPSHOT</version>
+ </parent>
+
+ <build>
+ <plugins>
+
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>1.9.1</version>
+ <executions>
+ <execution>
+ <id>attach-artifacts</id>
+ <phase>package</phase>
+ <goals>
+ <goal>attach-artifact</goal>
+ </goals>
+ <configuration>
+ <artifacts>
+ <artifact>
+ <file>target/${project.artifactId}-${project.version}.apa</file>
+ <type>apa</type>
+ </artifact>
+ </artifacts>
+ <skipAttach>false</skipAttach>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ </plugins>
+ </build>
+
+ <dependencies>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.10</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <!-- required by twitter example -->
+ <groupId>org.twitter4j</groupId>
+ <artifactId>twitter4j-core</artifactId>
+ <version>4.0.4</version>
+ </dependency>
+ <dependency>
+ <!-- required by twitter example -->
+ <groupId>org.twitter4j</groupId>
+ <artifactId>twitter4j-stream</artifactId>
+ <version>4.0.4</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-contrib</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-stream</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.hsqldb</groupId>
+ <artifactId>hsqldb</artifactId>
+ <version>2.3.1</version>
+ </dependency>
+ <dependency>
+ <groupId>com.h2database</groupId>
+ <artifactId>h2</artifactId>
+ <version>1.4.192</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>joda-time</groupId>
+ <artifactId>joda-time</artifactId>
+ <version>2.9.1</version>
+ </dependency>
+ <dependency>
+ <!--This dependency is needed for StreamingWordExtractTest-->
+ <groupId>org.codehaus.janino</groupId>
+ <artifactId>commons-compiler</artifactId>
+ <version>2.7.8</version>
+ <type>jar</type>
+ </dependency>
+ <dependency>
+ <!--This dependency is needed for StreamingWordExtractTest-->
+ <groupId>org.codehaus.janino</groupId>
+ <artifactId>janino</artifactId>
+ <version>2.7.8</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+
+</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/highlevelapi/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/examples/highlevelapi/src/assemble/appPackage.xml b/examples/highlevelapi/src/assemble/appPackage.xml
new file mode 100644
index 0000000..4138cf2
--- /dev/null
+++ b/examples/highlevelapi/src/assemble/appPackage.xml
@@ -0,0 +1,59 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+ <id>appPackage</id>
+ <formats>
+ <format>jar</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>${basedir}/target/</directory>
+ <outputDirectory>/app</outputDirectory>
+ <includes>
+ <include>${project.artifactId}-${project.version}.jar</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/target/deps</directory>
+ <outputDirectory>/lib</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/site/conf</directory>
+ <outputDirectory>/conf</outputDirectory>
+ <includes>
+ <include>*.xml</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/META-INF</directory>
+ <outputDirectory>/META-INF</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/app</directory>
+ <outputDirectory>/app</outputDirectory>
+ </fileSet>
+ </fileSets>
+
+</assembly>
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/MinimalWordCount.java
----------------------------------------------------------------------
diff --git a/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/MinimalWordCount.java b/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/MinimalWordCount.java
new file mode 100644
index 0000000..327c882
--- /dev/null
+++ b/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/MinimalWordCount.java
@@ -0,0 +1,128 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.malhar.stream.sample;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.apex.malhar.lib.window.TriggerOption;
+import org.apache.apex.malhar.lib.window.Tuple;
+import org.apache.apex.malhar.lib.window.WindowOption;
+import org.apache.apex.malhar.stream.api.function.Function;
+import org.apache.apex.malhar.stream.api.impl.StreamFactory;
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.Context;
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.DefaultInputPort;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.common.util.BaseOperator;
+import com.datatorrent.lib.util.KeyValPair;
+
+import static org.apache.apex.malhar.stream.api.Option.Options.name;
+
+/**
+ * Beam MinimalWordCount Example
+ *
+ * @since 3.5.0
+ */
+@ApplicationAnnotation(name = "MinimalWordCount")
+public class MinimalWordCount implements StreamingApplication
+{
+ public static class Collector extends BaseOperator
+ {
+ static Map<String, Long> result;
+ private static boolean done = false;
+
+ public static boolean isDone()
+ {
+ return done;
+ }
+
+ @Override
+ public void setup(Context.OperatorContext context)
+ {
+ done = false;
+ result = new HashMap<>();
+ }
+
+ public final transient DefaultInputPort<KeyValPair<String, Long>> input = new DefaultInputPort<KeyValPair<String, Long>>()
+ {
+ @Override
+ public void process(KeyValPair<String, Long> tuple)
+ {
+ if (tuple.getKey().equals("bye")) {
+ done = true;
+ }
+ result.put(tuple.getKey(), tuple.getValue());
+ }
+ };
+ }
+
+ /**
+ * Populate the dag using High-Level API.
+ * @param dag
+ * @param conf
+ */
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ Collector collector = new Collector();
+ // Create a stream reading from a file line by line using StreamFactory.
+ StreamFactory.fromFolder("./src/test/resources/wordcount", name("textInput"))
+ // Use a flatmap transformation to extract words from the incoming stream of lines.
+ .flatMap(new Function.FlatMapFunction<String, String>()
+ {
+ @Override
+ public Iterable<String> f(String input)
+ {
+ return Arrays.asList(input.split("[^a-zA-Z']+"));
+
+ }
+ }, name("ExtractWords"))
+ // Apply windowing to the stream for counting, in this case, the window option is global window.
+ .window(new WindowOption.GlobalWindow(), new TriggerOption().accumulatingFiredPanes().withEarlyFiringsAtEvery(1))
+ // Count the appearances of every word.
+ .countByKey(new Function.ToKeyValue<String, String, Long>()
+ {
+ @Override
+ public Tuple<KeyValPair<String, Long>> f(String input)
+ {
+ return new Tuple.PlainTuple<KeyValPair<String, Long>>(new KeyValPair<String, Long>(input, 1L));
+ }
+ }, name("countByKey"))
+ // Format the counting result to a readable format by unwrapping the tuples.
+ .map(new Function.MapFunction<Tuple.WindowedTuple<KeyValPair<String, Long>>, KeyValPair<String, Long>>()
+ {
+ @Override
+ public KeyValPair<String, Long> f(Tuple.WindowedTuple<KeyValPair<String, Long>> input)
+ {
+ return input.getValue();
+ }
+ }, name("FormatResults"))
+ // Print the result.
+ .print(name("console"))
+ // Attach a collector to the stream to collect results.
+ .endWith(collector, collector.input, name("Collector"))
+ // populate the dag using the stream.
+ .populateDag(dag);
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/WindowedWordCount.java
----------------------------------------------------------------------
diff --git a/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/WindowedWordCount.java b/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/WindowedWordCount.java
new file mode 100644
index 0000000..5b83bd0
--- /dev/null
+++ b/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/WindowedWordCount.java
@@ -0,0 +1,290 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.malhar.stream.sample;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.joda.time.Duration;
+import org.apache.apex.malhar.lib.window.TriggerOption;
+import org.apache.apex.malhar.lib.window.Tuple;
+import org.apache.apex.malhar.lib.window.WindowOption;
+import org.apache.apex.malhar.stream.api.ApexStream;
+import org.apache.apex.malhar.stream.api.WindowedStream;
+import org.apache.apex.malhar.stream.api.function.Function;
+import org.apache.apex.malhar.stream.api.impl.StreamFactory;
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
+
+import com.google.common.base.Throwables;
+import com.datatorrent.api.Context;
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.DefaultInputPort;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.InputOperator;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.common.util.BaseOperator;
+import com.datatorrent.lib.util.KeyValPair;
+
+import static org.apache.apex.malhar.stream.api.Option.Options.name;
+
+/**
+ * Beam WindowedWordCount Example.
+ *
+ * @since 3.5.0
+ */
+@ApplicationAnnotation(name = "WindowedWordCount")
+public class WindowedWordCount implements StreamingApplication
+{
+ static final int WINDOW_SIZE = 1; // Default window duration in minutes
+
+ /**
+ * A input operator that reads from and output a file line by line to downstream with a time gap between
+ * every two lines.
+ */
+ public static class TextInput extends BaseOperator implements InputOperator
+ {
+ public final transient DefaultOutputPort<String> output = new DefaultOutputPort<>();
+ private boolean done = false;
+
+ private transient BufferedReader reader;
+
+ @Override
+ public void setup(Context.OperatorContext context)
+ {
+ done = false;
+ initReader();
+ }
+
+ private void initReader()
+ {
+ try {
+ InputStream resourceStream = this.getClass().getResourceAsStream("/wordcount/word.txt");
+ reader = new BufferedReader(new InputStreamReader(resourceStream));
+ } catch (Exception ex) {
+ throw Throwables.propagate(ex);
+ }
+ }
+
+ @Override
+ public void teardown()
+ {
+ IOUtils.closeQuietly(reader);
+ }
+
+ @Override
+ public void emitTuples()
+ {
+ if (!done) {
+ try {
+ String line = reader.readLine();
+ if (line == null) {
+ done = true;
+ reader.close();
+ } else {
+ this.output.emit(line);
+ }
+ Thread.sleep(50);
+ } catch (IOException ex) {
+ throw new RuntimeException(ex);
+ } catch (InterruptedException e) {
+ throw Throwables.propagate(e);
+ }
+ }
+ }
+ }
+
+ public static class Collector extends BaseOperator
+ {
+ private static Map<KeyValPair<Long, String>, Long> result = new HashMap<>();
+ private static boolean done = false;
+
+ @Override
+ public void setup(Context.OperatorContext context)
+ {
+ super.setup(context);
+ done = false;
+ }
+
+ public static boolean isDone()
+ {
+ return done;
+ }
+
+ public static Map<KeyValPair<Long, String>, Long> getResult()
+ {
+ return result;
+ }
+
+ public final transient DefaultInputPort<PojoEvent> input = new DefaultInputPort<PojoEvent>()
+ {
+ @Override
+ public void process(PojoEvent tuple)
+ {
+ result.put(new KeyValPair<Long, String>(tuple.getTimestamp(), tuple.getWord()), tuple.getCount());
+ if (tuple.getWord().equals("bye")) {
+ done = true;
+ }
+ }
+ };
+ }
+
+ /**
+ * A Pojo Tuple class used for outputting result to JDBC.
+ */
+ public static class PojoEvent
+ {
+ private String word;
+ private long count;
+ private long timestamp;
+
+ @Override
+ public String toString()
+ {
+ return "PojoEvent (word=" + getWord() + ", count=" + getCount() + ", timestamp=" + getTimestamp() + ")";
+ }
+
+ public String getWord()
+ {
+ return word;
+ }
+
+ public void setWord(String word)
+ {
+ this.word = word;
+ }
+
+ public long getCount()
+ {
+ return count;
+ }
+
+ public void setCount(long count)
+ {
+ this.count = count;
+ }
+
+ public long getTimestamp()
+ {
+ return timestamp;
+ }
+
+ public void setTimestamp(long timestamp)
+ {
+ this.timestamp = timestamp;
+ }
+ }
+
+ /**
+ * A map function that wrap the input string with a random generated timestamp.
+ */
+ public static class AddTimestampFn implements Function.MapFunction<String, Tuple.TimestampedTuple<String>>
+ {
+ private static final Duration RAND_RANGE = Duration.standardMinutes(10);
+ private final Long minTimestamp;
+
+ AddTimestampFn()
+ {
+ this.minTimestamp = System.currentTimeMillis();
+ }
+
+ @Override
+ public Tuple.TimestampedTuple<String> f(String input)
+ {
+ // Generate a timestamp that falls somewhere in the past two hours.
+ long randMillis = (long)(Math.random() * RAND_RANGE.getMillis());
+ long randomTimestamp = minTimestamp + randMillis;
+
+ return new Tuple.TimestampedTuple<>(randomTimestamp, input);
+ }
+ }
+
+ /** A MapFunction that converts a Word and Count into a PojoEvent. */
+ public static class FormatAsTableRowFn implements Function.MapFunction<Tuple.WindowedTuple<KeyValPair<String, Long>>, PojoEvent>
+ {
+ @Override
+ public PojoEvent f(Tuple.WindowedTuple<KeyValPair<String, Long>> input)
+ {
+ PojoEvent row = new PojoEvent();
+ row.setTimestamp(input.getTimestamp());
+ row.setCount(input.getValue().getValue());
+ row.setWord(input.getValue().getKey());
+ return row;
+ }
+ }
+
+ /**
+ * Populate dag with High-Level API.
+ * @param dag
+ * @param conf
+ */
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ TextInput input = new TextInput();
+ Collector collector = new Collector();
+
+ // Create stream from the TextInput operator.
+ ApexStream<Tuple.TimestampedTuple<String>> stream = StreamFactory.fromInput(input, input.output, name("input"))
+
+ // Extract all the words from the input line of text.
+ .flatMap(new Function.FlatMapFunction<String, String>()
+ {
+ @Override
+ public Iterable<String> f(String input)
+ {
+ return Arrays.asList(input.split("[\\p{Punct}\\s]+"));
+ }
+ }, name("ExtractWords"))
+
+ // Wrap the word with a randomly generated timestamp.
+ .map(new AddTimestampFn(), name("AddTimestampFn"));
+
+
+ // apply window and trigger option.
+ // TODO: change trigger option to atWaterMark when available.
+ WindowedStream<Tuple.TimestampedTuple<String>> windowedWords = stream
+ .window(new WindowOption.TimeWindows(Duration.standardMinutes(WINDOW_SIZE)),
+ new TriggerOption().accumulatingFiredPanes().withEarlyFiringsAtEvery(1));
+
+
+ WindowedStream<PojoEvent> wordCounts =
+ // Perform a countByKey transformation to count the appearance of each word in every time window.
+ windowedWords.countByKey(new Function.ToKeyValue<Tuple.TimestampedTuple<String>, String, Long>()
+ {
+ @Override
+ public Tuple<KeyValPair<String, Long>> f(Tuple.TimestampedTuple<String> input)
+ {
+ return new Tuple.TimestampedTuple<KeyValPair<String, Long>>(input.getTimestamp(),
+ new KeyValPair<String, Long>(input.getValue(), 1L));
+ }
+ }, name("count words"))
+
+ // Format the output and print out the result.
+ .map(new FormatAsTableRowFn(), name("FormatAsTableRowFn")).print(name("console"));
+
+ wordCounts.endWith(collector, collector.input, name("Collector")).populateDag(dag);
+ }
+}
[25/30] apex-malhar git commit: Renamed demos to examples. Packages
and artifactid names are changed as suggested.
Posted by th...@apache.org.
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/cookbook/MaxPerKeyExamplesTest.java
----------------------------------------------------------------------
diff --git a/demos/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/cookbook/MaxPerKeyExamplesTest.java b/demos/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/cookbook/MaxPerKeyExamplesTest.java
deleted file mode 100644
index ec28b40..0000000
--- a/demos/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/cookbook/MaxPerKeyExamplesTest.java
+++ /dev/null
@@ -1,210 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.apex.malhar.stream.sample.cookbook;
-
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.Callable;
-
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import org.apache.hadoop.conf.Configuration;
-
-import com.google.common.base.Throwables;
-
-import com.datatorrent.api.LocalMode;
-import com.datatorrent.lib.db.jdbc.JdbcTransactionalStore;
-import com.datatorrent.stram.StramLocalCluster;
-
-/**
- * Test for MaxPerKeyExamples Application.
- */
-public class MaxPerKeyExamplesTest
-{
-
- private static final String INPUT_TUPLE_CLASS = "org.apache.apex.malhar.stream.sample.cookbook.InputPojo";
- private static final String OUTPUT_TUPLE_CLASS = "org.apache.apex.malhar.stream.sample.cookbook.OutputPojo";
- private static final String DB_DRIVER = "org.h2.Driver";
- private static final String DB_URL = "jdbc:h2:~/test";
- private static final String INPUT_TABLE = "InputTable";
- private static final String OUTPUT_TABLE = "OutputTable";
- private static final String USER_NAME = "root";
- private static final String PSW = "password";
- private static final String QUERY = "SELECT * FROM " + INPUT_TABLE + ";";
-
- private static final double[] MEANTEMPS = {85.3, 75.4};
-
- @BeforeClass
- public static void setup()
- {
- try {
- Class.forName(DB_DRIVER).newInstance();
-
- Connection con = DriverManager.getConnection(DB_URL,USER_NAME,PSW);
- Statement stmt = con.createStatement();
-
- String createMetaTable = "CREATE TABLE IF NOT EXISTS " + JdbcTransactionalStore.DEFAULT_META_TABLE + " ( "
- + JdbcTransactionalStore.DEFAULT_APP_ID_COL + " VARCHAR(100) NOT NULL, "
- + JdbcTransactionalStore.DEFAULT_OPERATOR_ID_COL + " INT NOT NULL, "
- + JdbcTransactionalStore.DEFAULT_WINDOW_COL + " BIGINT NOT NULL, "
- + "UNIQUE (" + JdbcTransactionalStore.DEFAULT_APP_ID_COL + ", "
- + JdbcTransactionalStore.DEFAULT_OPERATOR_ID_COL + ", " + JdbcTransactionalStore.DEFAULT_WINDOW_COL + ") "
- + ")";
- stmt.executeUpdate(createMetaTable);
-
- String createInputTable = "CREATE TABLE IF NOT EXISTS " + INPUT_TABLE
- + "(MONTH INT(2) not NULL, DAY INT(2), YEAR INT(4), MEANTEMP DOUBLE(10) )";
- stmt.executeUpdate(createInputTable);
-
- String createOutputTable = "CREATE TABLE IF NOT EXISTS " + OUTPUT_TABLE
- + "(MONTH INT(2) not NULL, MEANTEMP DOUBLE(10) )";
- stmt.executeUpdate(createOutputTable);
-
- String cleanTable = "truncate table " + INPUT_TABLE;
- stmt.executeUpdate(cleanTable);
-
- stmt = con.createStatement();
-
- String sql = "INSERT INTO " + INPUT_TABLE + " VALUES (6, 21, 2014, 85.3)";
- stmt.executeUpdate(sql);
- sql = "INSERT INTO " + INPUT_TABLE + " VALUES (7, 20, 2014, 75.4)";
- stmt.executeUpdate(sql);
- sql = "INSERT INTO " + INPUT_TABLE + " VALUES (6, 18, 2014, 45.3)";
- stmt.executeUpdate(sql);
-
- } catch (Throwable e) {
- throw Throwables.propagate(e);
- }
- }
-
- @AfterClass
- public static void cleanup()
- {
- try {
- Class.forName(DB_DRIVER).newInstance();
-
- Connection con = DriverManager.getConnection(DB_URL, USER_NAME, PSW);
- Statement stmt = con.createStatement();
-
- String dropInputTable = "DROP TABLE " + INPUT_TABLE;
- stmt.executeUpdate(dropInputTable);
-
- String dropOutputTable = "DROP TABLE " + OUTPUT_TABLE;
- stmt.executeUpdate(dropOutputTable);
-
- } catch (Throwable e) {
- throw Throwables.propagate(e);
- }
-
- }
-
- public void setConfig(Configuration conf)
- {
- conf.set("dt.operator.jdbcInput.prop.store.userName", USER_NAME);
- conf.set("dt.operator.jdbcInput.prop.store.password", PSW);
- conf.set("dt.operator.jdbcInput.prop.store.databaseDriver", DB_DRIVER);
- conf.set("dt.operator.jdbcInput.prop.batchSize", "5");
- conf.set("dt.operator.jdbcInput.port.outputPort.attr.TUPLE_CLASS", INPUT_TUPLE_CLASS);
- conf.set("dt.operator.jdbcInput.prop.store.databaseUrl", DB_URL);
- conf.set("dt.operator.jdbcInput.prop.tableName", INPUT_TABLE);
- conf.set("dt.operator.jdbcInput.prop.query", QUERY);
-
- conf.set("dt.operator.jdbcOutput.prop.store.userName", USER_NAME);
- conf.set("dt.operator.jdbcOutput.prop.store.password", PSW);
- conf.set("dt.operator.jdbcOutput.prop.store.databaseDriver", DB_DRIVER);
- conf.set("dt.operator.jdbcOutput.prop.batchSize", "5");
- conf.set("dt.operator.jdbcOutput.port.input.attr.TUPLE_CLASS", OUTPUT_TUPLE_CLASS);
- conf.set("dt.operator.jdbcOutput.prop.store.databaseUrl", DB_URL);
- conf.set("dt.operator.jdbcOutput.prop.tablename", OUTPUT_TABLE);
- }
-
- public int getNumEntries()
- {
- Connection con;
- try {
- con = DriverManager.getConnection(DB_URL,USER_NAME,PSW);
- Statement stmt = con.createStatement();
-
- String countQuery = "SELECT count(DISTINCT (MONTH, MEANTEMP)) from " + OUTPUT_TABLE;
- ResultSet resultSet = stmt.executeQuery(countQuery);
- resultSet.next();
- return resultSet.getInt(1);
- } catch (SQLException e) {
- throw new RuntimeException("fetching count", e);
- }
- }
-
- public Map<Integer, Double> getMaxMeanTemp()
- {
- Map<Integer, Double> result = new HashMap<>();
- Connection con;
- try {
- con = DriverManager.getConnection(DB_URL,USER_NAME,PSW);
- Statement stmt = con.createStatement();
-
- String countQuery = "SELECT DISTINCT * from " + OUTPUT_TABLE;
- ResultSet resultSet = stmt.executeQuery(countQuery);
- while (resultSet.next()) {
- result.put(resultSet.getInt("MONTH"), resultSet.getDouble("MEANTEMP"));
-
- }
- return result;
- } catch (SQLException e) {
- throw new RuntimeException("fetching count", e);
- }
- }
-
- @Test
- public void MaxPerKeyExampleTest() throws Exception
- {
- LocalMode lma = LocalMode.newInstance();
- Configuration conf = new Configuration(false);
- setConfig(conf);
-
- MaxPerKeyExamples app = new MaxPerKeyExamples();
-
- lma.prepareDAG(app, conf);
-
- LocalMode.Controller lc = lma.getController();
- ((StramLocalCluster)lc).setExitCondition(new Callable<Boolean>()
- {
- @Override
- public Boolean call() throws Exception
- {
- return getNumEntries() == 2;
- }
- });
-
- lc.run(5000);
-
- double[] result = new double[2];
- result[0] = getMaxMeanTemp().get(6);
- result[1] = getMaxMeanTemp().get(7);
- Assert.assertArrayEquals(MEANTEMPS, result, 0.0);
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/highlevelapi/src/test/resources/data/word.txt
----------------------------------------------------------------------
diff --git a/demos/highlevelapi/src/test/resources/data/word.txt b/demos/highlevelapi/src/test/resources/data/word.txt
deleted file mode 100644
index 7e28409..0000000
--- a/demos/highlevelapi/src/test/resources/data/word.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-word1 word2 word7 word9 word2 word3 word2 word4 word7 word3 word9 word9 word5 word5 word4 word2 word1 error
-word1 word2 word7 word9 word2 word3 word2 word4 word7 word3 word9 word9 word5 word5 word4 word2 word1 error
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/highlevelapi/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/demos/highlevelapi/src/test/resources/log4j.properties b/demos/highlevelapi/src/test/resources/log4j.properties
deleted file mode 100644
index 592eb19..0000000
--- a/demos/highlevelapi/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,45 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-log4j.rootLogger=INFO,CONSOLE
-
-log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
-log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
-log4j.appender.CONSOLE.threshold=INFO
-#log4j.appender.CONSOLE.threshold=${test.log.console.threshold}
-test.log.console.threshold=WARN
-
-log4j.appender.RFA=org.apache.log4j.RollingFileAppender
-log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
-log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
-log4j.appender.RFA.File=/tmp/app.log
-
-# to enable, add SYSLOG to rootLogger
-log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender
-log4j.appender.SYSLOG.syslogHost=127.0.0.1
-log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout
-log4j.appender.SYSLOG.layout.conversionPattern=${dt.cid} %-5p [%t] %c{2} %x - %m%n
-log4j.appender.SYSLOG.Facility=LOCAL1
-
-#log4j.logger.org=INFO
-
-#log4j.logger.org.apache.commons.beanutils=warn
-log4j.logger.com.datatorrent=INFO
-log4j.logger.org.apache.apex=INFO
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/highlevelapi/src/test/resources/sampletweets.txt
----------------------------------------------------------------------
diff --git a/demos/highlevelapi/src/test/resources/sampletweets.txt b/demos/highlevelapi/src/test/resources/sampletweets.txt
deleted file mode 100644
index c113130..0000000
--- a/demos/highlevelapi/src/test/resources/sampletweets.txt
+++ /dev/null
@@ -1,44 +0,0 @@
-Last week, I published a blog announcing that Apex was accepted as an Apache incubator project.
-This week, I\u2019ll give you a little more detail on what Apache Apex is, and why it\u2019s important.
-
-Apache #Hadoop has been around for over a decade. It has become the de-facto big data platform,
-allowing enterprises to transform their business operations by turning big data into something useful, meaningful,
-and revenue-generating. #Hadoop promised the enablement of big data without incurring the costs you would normally think such powerful
-processing systems would demand. This tremendous promise of transforming business operations continues to fuel high growth in the industry.
-
-It all got started when Hadoop engineers at Yahoo! asked, \u201cHow can we build an efficient search indexing capability?\u201d
-The ensuing iterations and some inspiration resulted in the #MapReduce programming model. Although powerful, MapReduce wasn\u2019t perfect.
-
-Mastering MapReduce required a steep learning curve. Migrating applications to MapReduce required an almost complete rewrite.
-Equally worrisome was the fact that MapReduce had batch processing paradigm and \u201ccompute going to data\u201d at its core,
-thus posing a deterrent to Hadoop realizing its true potential.
-
-Expectedly enough, #MapReduce was an impediment that did little to bolster productization of big data.
-Not to be deterred, there were faster substitutes for MapReduce. Just like Hadoop, these models required deeper expertise, were tough to operate and difficult to master.
-As such, #Hadoop disrupted the way big data needs were handled, but remained largely under-productized.
-
-A decade after Hadoop was started, only a small percentage of big data projects are in production.
-Data is growing rapidly and the ability to harness big data has become a decisive competitive advantage.
-MapReduce impedes this demand (actually more of a scramble) to transform into a data-driven business.
-
-In hindsight, it is clear that in the early days, the subsequent success of Hadoop was not anticipated.
-If they had anticipated Hadoop\u2019s success, the question would have been, \u201cWhat can we do with massively distributed resources?\u201d
-The answer to this question, which came about soon after, was YARN (Hadoop 2.0), the next generation Hadoop.
-For the first time, #YARN brought the capability of exploring how distributed resources handling big data could perform \u201ca lot of things\u201d,
-thus going beyond the early MapReduce paradigm, and in a way beyond batch or even compute-going-to-data paradigms.
-YARN presented the capability to allow big data to not just become big in size, but broader in use cases. With its enabling capability as a Hadoop facilitator,
-YARN has pushed Hadoop towards realizing its true potential. The Hadoop predicament is similar to what
-cellphones would have been without the more popular features such as messaging and internet connectivity.
-
-
-In their early years, cellphones upset the landline market, but did not foster an immediate market furor till
-it transformed into the new-age \u201csmartphone\u201d with impressive features.
-YARN is most certainly the enabling factor for big data dreaming bigger and wider, and with it, Hadoop 2.0 is now a true de-facto distributed operating system.
-
-What\u2019s needed is bleeding edge YARN-based platforms capable of radically realizing Hadoop\u2019s potential
-
-Now is the right time to not only productize big data, but to see how setting it in motion can ensure realization of greater business goals.
-A Herculean task, this demands platforms that are easy to deploy, require nothing beyond everyday IT expertise, can effortlessly integrate with an existing IT infrastructure while ensuring ease of migration.
-The new-age Hadoop platforms need to be designed with an approach to reduce time-to-market by shortening the application lifecycle, from building to launching, thus quickening the realization of revenue for businesses.
-They will also have to reduce time for developers to develop, devOps to operationalize, and finally reduce time to insight for business.
-Platforms such as these will need to learn, adapt, and change to meet the burgeoning needs of the big data world.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/highlevelapi/src/test/resources/wordcount/word.txt
----------------------------------------------------------------------
diff --git a/demos/highlevelapi/src/test/resources/wordcount/word.txt b/demos/highlevelapi/src/test/resources/wordcount/word.txt
deleted file mode 100644
index edd0f51..0000000
--- a/demos/highlevelapi/src/test/resources/wordcount/word.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-word1 word2 word7 word9 word2 word3 word2 word4 word7 word3 word9 word9 word5 word5 word4 word2 word1 error
-word1 word2 word7 word9 word2 word3 word2 word4 word7 word3 word9 word9 word5 word5 word4 word2 word1 error
-word1 word2 word7 word9 word2 word3 word2 word4 word7 word3 word9 word9 word5 word5 word4 word2 word1 error
-word1 word2 word7 word9 word2 word3 word2 word4 word7 word3 word9 word9 word5 word5 word4 word2 word1 error
-word1 word2 word7 word9 word2 word3 word2 word4 word7 word3 word9 word9 word5 word5 word4 word2 word1 error
-word1 word2 word7 word9 word2 word3 word2 word4 word7 word3 word9 word9 word5 word5 word4 word2 word1 error
-word1 word2 word7 word9 word2 word3 word2 word4 word7 word3 word9 word9 word5 word5 word4 word2 word1 error
-bye
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/iteration/pom.xml
----------------------------------------------------------------------
diff --git a/demos/iteration/pom.xml b/demos/iteration/pom.xml
deleted file mode 100644
index 1e2fe7c..0000000
--- a/demos/iteration/pom.xml
+++ /dev/null
@@ -1,37 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>iteration-demo</artifactId>
- <packaging>jar</packaging>
-
- <name>Apache Apex Malhar Iteration Demo</name>
- <description>DataTorrent demo applications that demonstrates the iteration feature.</description>
-
- <parent>
- <groupId>org.apache.apex</groupId>
- <artifactId>malhar-demos</artifactId>
- <version>3.7.0-SNAPSHOT</version>
- </parent>
-
-</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/iteration/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/demos/iteration/src/assemble/appPackage.xml b/demos/iteration/src/assemble/appPackage.xml
deleted file mode 100644
index 4138cf2..0000000
--- a/demos/iteration/src/assemble/appPackage.xml
+++ /dev/null
@@ -1,59 +0,0 @@
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
- <id>appPackage</id>
- <formats>
- <format>jar</format>
- </formats>
- <includeBaseDirectory>false</includeBaseDirectory>
- <fileSets>
- <fileSet>
- <directory>${basedir}/target/</directory>
- <outputDirectory>/app</outputDirectory>
- <includes>
- <include>${project.artifactId}-${project.version}.jar</include>
- </includes>
- </fileSet>
- <fileSet>
- <directory>${basedir}/target/deps</directory>
- <outputDirectory>/lib</outputDirectory>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/site/conf</directory>
- <outputDirectory>/conf</outputDirectory>
- <includes>
- <include>*.xml</include>
- </includes>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/main/resources/META-INF</directory>
- <outputDirectory>/META-INF</outputDirectory>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/main/resources/app</directory>
- <outputDirectory>/app</outputDirectory>
- </fileSet>
- </fileSets>
-
-</assembly>
-
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/iteration/src/main/java/com/datatorrent/demos/iteration/Application.java
----------------------------------------------------------------------
diff --git a/demos/iteration/src/main/java/com/datatorrent/demos/iteration/Application.java b/demos/iteration/src/main/java/com/datatorrent/demos/iteration/Application.java
deleted file mode 100644
index 7fbdfd1..0000000
--- a/demos/iteration/src/main/java/com/datatorrent/demos/iteration/Application.java
+++ /dev/null
@@ -1,171 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.iteration;
-
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.PrintStream;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.Context;
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.DefaultInputPort;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-import com.datatorrent.common.util.BaseOperator;
-import com.datatorrent.common.util.DefaultDelayOperator;
-import com.datatorrent.lib.testbench.RandomEventGenerator;
-
-/**
- * Iteration demo : <br>
- *
- * <pre>
- * LocalMode.runApp(new Application(), 600000); // 10 min run
- * </pre>
- *
- * Run Success : <br>
- * For successful deployment and run, user should see the Fibonacci sequence, something like the
- * following output on the console:
- *
- * <pre>
- * 1
- * 1
- * 2
- * 3
- * 5
- * 8
- * 13
- * 21
- * 34
- * 55
- * ...
- * </pre>
- *
- *
- * @since 3.4.0
- */
-@ApplicationAnnotation(name = "IterationDemo")
-public class Application implements StreamingApplication
-{
- private static final Logger LOG = LoggerFactory.getLogger(Application.class);
- private String extraOutputFileName; // for unit test
-
- public static class FibonacciOperator extends BaseOperator
- {
- public long currentNumber = 1;
- private transient long tempNum;
- public transient DefaultInputPort<Object> dummyInputPort = new DefaultInputPort<Object>()
- {
- @Override
- public void process(Object tuple)
- {
- }
- };
- public transient DefaultInputPort<Long> input = new DefaultInputPort<Long>()
- {
- @Override
- public void process(Long tuple)
- {
- tempNum = (currentNumber == 1) ? 1 : tuple;
- }
- };
- public transient DefaultOutputPort<Long> output = new DefaultOutputPort<>();
-
-
- @Override
- public void endWindow()
- {
- output.emit(currentNumber);
- currentNumber += tempNum;
- if (currentNumber <= 0) {
- currentNumber = 1;
- }
- }
- }
-
- public static class StdoutOperator extends BaseOperator
- {
- private String extraOutputFileName; // for unit test
- private transient PrintStream extraOutputStream;
- /**
- * This is the input port which receives the tuples that will be written to stdout.
- */
- public final transient DefaultInputPort<Object> input = new DefaultInputPort<Object>()
- {
- @Override
- @SuppressWarnings("UseOfSystemOutOrSystemErr")
- public void process(Object t)
- {
- String s = t.toString();
- LOG.info(s);
- if (extraOutputStream != null) {
- extraOutputStream.println(s);
- }
- }
- };
-
- @Override
- public void setup(Context.OperatorContext context)
- {
- if (extraOutputFileName != null) {
- try {
- extraOutputStream = new PrintStream(new FileOutputStream(extraOutputFileName), true);
- } catch (IOException ex) {
- throw new RuntimeException(ex);
- }
- }
- }
-
- @Override
- public void teardown()
- {
- extraOutputStream.close();
- }
-
- public void setExtraOutputFileName(String fileName)
- {
- this.extraOutputFileName = fileName;
- }
- }
-
- public void setExtraOutputFileName(String fileName)
- {
- this.extraOutputFileName = fileName;
- }
-
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- RandomEventGenerator rand = dag.addOperator("rand", new RandomEventGenerator());
- FibonacciOperator fib = dag.addOperator("FIB", FibonacciOperator.class);
- DefaultDelayOperator opDelay = dag.addOperator("opDelay", DefaultDelayOperator.class);
- StdoutOperator console = new StdoutOperator();
- console.setExtraOutputFileName(extraOutputFileName);
- dag.addOperator("console", console);
- dag.addStream("dummy_to_operator", rand.integer_data, fib.dummyInputPort);
- dag.addStream("operator_to_delay", fib.output, opDelay.input, console.input);
- dag.addStream("delay_to_operator", opDelay.output, fib.input);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/iteration/src/main/java/com/datatorrent/demos/iteration/package-info.java
----------------------------------------------------------------------
diff --git a/demos/iteration/src/main/java/com/datatorrent/demos/iteration/package-info.java b/demos/iteration/src/main/java/com/datatorrent/demos/iteration/package-info.java
deleted file mode 100644
index d0b62ea..0000000
--- a/demos/iteration/src/main/java/com/datatorrent/demos/iteration/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/**
- * Iteration demonstration application.
- */
-package com.datatorrent.demos.iteration;
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/iteration/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/demos/iteration/src/main/resources/META-INF/properties.xml b/demos/iteration/src/main/resources/META-INF/properties.xml
deleted file mode 100644
index 5212b91..0000000
--- a/demos/iteration/src/main/resources/META-INF/properties.xml
+++ /dev/null
@@ -1,45 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<configuration>
- <!-- Memory settings for all demos -->
- <property>
- <name>dt.attr.MASTER_MEMORY_MB</name>
- <value>512</value>
- </property>
- <property>
- <name>dt.attr.DEBUG</name>
- <value>true</value>
- </property>
- <property>
- <name>dt.application.*.operator.*.attr.MEMORY_MB</name>
- <value>128</value>
- </property>
- <property>
- <name>dt.application.*.operator.*.attr.JVM_OPTIONS</name>
- <value>-Xmx128M</value>
- </property>
- <property>
- <name>dt.application.*.operator.*.port.*.attr.BUFFER_MEMORY_MB</name>
- <value>128</value>
- </property>
-
-</configuration>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/iteration/src/test/java/com/datatorrent/demos/iteration/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/demos/iteration/src/test/java/com/datatorrent/demos/iteration/ApplicationTest.java b/demos/iteration/src/test/java/com/datatorrent/demos/iteration/ApplicationTest.java
deleted file mode 100644
index 9fb89ac..0000000
--- a/demos/iteration/src/test/java/com/datatorrent/demos/iteration/ApplicationTest.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.iteration;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileReader;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.LocalMode;
-
-/**
- *
- */
-public class ApplicationTest
-{
- @Test
- public void testIterationApp() throws Exception
- {
- LocalMode lma = LocalMode.newInstance();
- Configuration conf = new Configuration(false);
- Application app = new Application();
- String outputFileName = "target/output.txt";
- long timeout = 10 * 1000; // 10 seconds
-
- new File(outputFileName).delete();
- app.setExtraOutputFileName(outputFileName);
- lma.prepareDAG(app, conf);
- LocalMode.Controller lc = lma.getController();
- lc.runAsync();
-
- long startTime = System.currentTimeMillis();
- do {
- try {
- Thread.sleep(500);
- } catch (InterruptedException ex) {
- break;
- }
- File file = new File(outputFileName);
- if (file.length() > 50) {
- break;
- }
- }
- while (System.currentTimeMillis() - startTime < timeout);
-
- lc.shutdown();
- try (BufferedReader br = new BufferedReader(new FileReader(outputFileName))) {
- Assert.assertEquals("1", br.readLine());
- Assert.assertEquals("1", br.readLine());
- Assert.assertEquals("2", br.readLine());
- Assert.assertEquals("3", br.readLine());
- Assert.assertEquals("5", br.readLine());
- Assert.assertEquals("8", br.readLine());
- Assert.assertEquals("13", br.readLine());
- Assert.assertEquals("21", br.readLine());
- Assert.assertEquals("34", br.readLine());
- Assert.assertEquals("55", br.readLine());
- Assert.assertEquals("89", br.readLine());
- Assert.assertEquals("144", br.readLine());
- Assert.assertEquals("233", br.readLine());
- Assert.assertEquals("377", br.readLine());
- Assert.assertEquals("610", br.readLine());
- Assert.assertEquals("987", br.readLine());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/iteration/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/demos/iteration/src/test/resources/log4j.properties b/demos/iteration/src/test/resources/log4j.properties
deleted file mode 100644
index cf0d19e..0000000
--- a/demos/iteration/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,43 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-log4j.rootLogger=DEBUG,CONSOLE
-
-log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
-log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
-log4j.appender.CONSOLE.threshold=${test.log.console.threshold}
-test.log.console.threshold=DEBUG
-
-log4j.appender.RFA=org.apache.log4j.RollingFileAppender
-log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
-log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
-log4j.appender.RFA.File=/tmp/app.log
-
-# to enable, add SYSLOG to rootLogger
-log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender
-log4j.appender.SYSLOG.syslogHost=127.0.0.1
-log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout
-log4j.appender.SYSLOG.layout.conversionPattern=${dt.cid} %-5p [%t] %c{2} %x - %m%n
-log4j.appender.SYSLOG.Facility=LOCAL1
-
-log4j.logger.org=info
-#log4j.logger.org.apache.commons.beanutils=warn
-log4j.logger.com.datatorrent=debug
-log4j.logger.org.apache.apex=debug
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/machinedata/pom.xml
----------------------------------------------------------------------
diff --git a/demos/machinedata/pom.xml b/demos/machinedata/pom.xml
deleted file mode 100644
index 5d5bac2..0000000
--- a/demos/machinedata/pom.xml
+++ /dev/null
@@ -1,66 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>machinedata-demo</artifactId>
- <packaging>jar</packaging>
-
- <name>Apache Apex Malhar MachineData Demo</name>
- <description></description>
-
- <parent>
- <groupId>org.apache.apex</groupId>
- <artifactId>malhar-demos</artifactId>
- <version>3.7.0-SNAPSHOT</version>
- </parent>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.apex</groupId>
- <artifactId>malhar-contrib</artifactId>
- <version>${project.version}</version>
- <exclusions>
- <exclusion>
- <groupId>*</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>redis.clients</groupId>
- <artifactId>jedis</artifactId>
- <version>2.5.1</version>
- </dependency>
- <dependency>
- <groupId>javax.mail</groupId>
- <artifactId>javax.mail-api</artifactId>
- <version>1.5.2</version>
- </dependency>
- <dependency>
- <groupId>com.sun.mail</groupId>
- <artifactId>javax.mail</artifactId>
- <version>1.5.2</version>
- </dependency>
- </dependencies>
-
-</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/machinedata/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/demos/machinedata/src/assemble/appPackage.xml b/demos/machinedata/src/assemble/appPackage.xml
deleted file mode 100644
index 4138cf2..0000000
--- a/demos/machinedata/src/assemble/appPackage.xml
+++ /dev/null
@@ -1,59 +0,0 @@
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
- <id>appPackage</id>
- <formats>
- <format>jar</format>
- </formats>
- <includeBaseDirectory>false</includeBaseDirectory>
- <fileSets>
- <fileSet>
- <directory>${basedir}/target/</directory>
- <outputDirectory>/app</outputDirectory>
- <includes>
- <include>${project.artifactId}-${project.version}.jar</include>
- </includes>
- </fileSet>
- <fileSet>
- <directory>${basedir}/target/deps</directory>
- <outputDirectory>/lib</outputDirectory>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/site/conf</directory>
- <outputDirectory>/conf</outputDirectory>
- <includes>
- <include>*.xml</include>
- </includes>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/main/resources/META-INF</directory>
- <outputDirectory>/META-INF</outputDirectory>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/main/resources/app</directory>
- <outputDirectory>/app</outputDirectory>
- </fileSet>
- </fileSets>
-
-</assembly>
-
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/machinedata/src/main/html/global.js
----------------------------------------------------------------------
diff --git a/demos/machinedata/src/main/html/global.js b/demos/machinedata/src/main/html/global.js
deleted file mode 100644
index 753f58f..0000000
--- a/demos/machinedata/src/main/html/global.js
+++ /dev/null
@@ -1,269 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/**
- * Declaration and initialization for global variables.
- */
-
-// url parameters
-var params;
-
-// Data Points
-var aggrData;
-var aggrDataPoints;
-var contData;
-var contDataPoints;
-
-// CPU data table
-var cpuTable;
-var cpuChart;
-var cpuView;
-
-// ram data table
-var ramTable;
-var ramChart;
-var ramView;
-
-// hdd data table
-var hddTable;
-var hddChart;
-var hddView;
-
-// chart options
-var chartOptions;
-
-// Date formatter
-var dateFormatter;
-
-// window look back value
-var lookback;
-var aggrLookBack;
-var contLookBack;
-var contRefresh;
-
-// Get split query string
-function QueryString() {
- var query_string = {};
- var query = window.location.search.substring(1);
- return query;
-}
-function SplitQuery(query)
-{
- var params = {};
- var vars = query.split("&");
- for (var i=0;i<vars.length;i++)
- {
- var pair = vars[i].split("=");
- if(pair.length == 2)
- {
- params[pair[0]] = pair[1];
- }
- }
- return params;
-}
-
-// Initialize global variable(s)
-function InitializeGlobal()
-{
- // Initialize params
- params = SplitQuery(QueryString());
-
- // Initialize data points
- aggrDataPoints = new Array();
- contDataPoints = new Array();
-
- // Initialize cpu table
- cpuTable = new google.visualization.DataTable();
- cpuTable.addColumn('datetime', 'Time');
- cpuTable.addColumn('number', 'CPU');
- chartOptions = { width: 600, height: 300, legend: 'none', pointSize: 0, lineWidth : 1 };
- cpuChart = new google.visualization.ScatterChart(document.getElementById('chart_div'));
- cpuView = new google.visualization.DataView(cpuTable);
-
- // Initialize ram table
- ramTable = new google.visualization.DataTable();
- ramTable.addColumn('datetime', 'Time');
- ramTable.addColumn('number', 'RAM');;
- ramChart = new google.visualization.ScatterChart(document.getElementById('chart1_div'));
- ramView = new google.visualization.DataView(ramTable);
-
- // Initialize hdd table
- hddTable = new google.visualization.DataTable();
- hddTable.addColumn('datetime', 'Time');
- hddTable.addColumn('number', 'HDD');;
- hddChart = new google.visualization.ScatterChart(document.getElementById('chart2_div'));
- hddView = new google.visualization.DataView(hddTable);
-
- // get lookback value
- lookback = (new Date().getTime()/1000) - 3600*6;
- if (params['lookback'] && (params['lookback'].length > 0)) lookback = (new Date().getTime()/1000) - (3600*(parseInt(params['lookback'])));
- aggrLookBack = lookback;
-
- // get continuos lookback
- contLookBack = lookback;
- contRefresh = 5;
-
- // get param lookback
- paramLookBack = 6;
- if (params['lookback'] && (params['lookback'].length > 0)) paramLookBack = parseInt(params['lookback']);
- //if (params['refresh'] && (params['refresh'].length > 0)) contRefresh = parseInt(params['refresh']);
-}
-
-
-/**
- * Function to create fetch urls from given parameters
- */
-function DataUrl()
-{
- var url = "json.php?bucket=m";
- url += "&customer=";
- if (params['customer'])
- {
- url += params['customer'];
- }
- url += "&product=";
- if (params['product'])
- {
- url += params['product'];
- }
- url += "&os=";
- if (params['os'])
- {
- url += params['os'];
- }
- url += "&software1=";
- if (params['software1'])
- {
- url += params['software1'];
- }
- url += "&software2=";
- if (params['software2'])
- {
- url += params['software2'];
- }
- url += "&software3=";
- if (params['software3'])
- {
- url += params['software3'];
- }
- url += "&from=";
- url += Math.floor(lookback);
- return url;
-}
-
-/**
- * Creates data table with time stamp and cpu values.
- * Draw line chart for time vs cpu.
- */
-function DrawCPUChart()
-{
- // create/delete rows
- if (cpuTable.getNumberOfRows() < aggrDataPoints.length)
- {
- var numRows = aggrDataPoints.length - cpuTable.getNumberOfRows();
- cpuTable.addRows(numRows);
- } else {
- for(var i=(cpuTable.getNumberOfRows()-1); i >= aggrDataPoints.length; i--)
- {
- cpuTable.removeRow(i);
- }
- }
-
- // Populate data table with time/cpu data points.
- for(var i=0; i < cpuTable.getNumberOfRows(); i++)
- {
- //if(parseFloat(aggrDataPoints[i].cpu) < 500) continue;
- cpuTable.setCell(i, 0, new Date(parseInt(aggrDataPoints[i].timestamp)));
- cpuTable.setCell(i, 1, parseFloat(aggrDataPoints[i].cpu));
- }
-
- // Draw line chart.
- chartOptions.title = 'CPU Usage (%)';
- cpuChart.draw(cpuView, chartOptions);
-}
-
-/**
- * Creates data table with time stamp and revenu values.
- * Draw line chart for time vs ram.
- */
-function DrawRAMChart()
-{
- // create/delete rows
- if (ramTable.getNumberOfRows() < aggrDataPoints.length)
- {
- var numRows = aggrDataPoints.length - ramTable.getNumberOfRows();
- ramTable.addRows(numRows);
- } else {
- for(var i=(ramTable.getNumberOfRows()-1); i >= aggrDataPoints.length; i--)
- {
- ramTable.removeRow(i);
- }
- }
-
- // Populate data table with time/ram data points.
- for(var i=0; i < ramTable.getNumberOfRows(); i++)
- {
- ramTable.setCell(i, 0, new Date(parseInt(aggrDataPoints[i].timestamp)));
- ramTable.setCell(i, 1, parseFloat(aggrDataPoints[i].ram));
- }
-
- // Draw line chart.
- chartOptions.title = 'RAM Usage (%)';
- ramChart.draw(ramView, chartOptions);
-}
-
-/**
- * Creates data table with time stamp and hdd values.
- * Draw line chart for time vs hdd.
- */
-function DrawHDDChart()
-{
- // create/delete rows
- if (hddTable.getNumberOfRows() < aggrDataPoints.length)
- {
- var numRows = aggrDataPoints.length - hddTable.getNumberOfRows();
- hddTable.addRows(numRows);
- } else {
- for(var i=(hddTable.getNumberOfRows()-1); i >= aggrDataPoints.length; i--)
- {
- hddTable.removeRow(i);
- }
- }
-
- // Populate data table with time/hdd data points.
- for(var i=0; i < hddTable.getNumberOfRows(); i++)
- {
- hddTable.setCell(i, 0, new Date(parseInt(aggrDataPoints[i].timestamp)));
- hddTable.setCell(i, 1, parseInt(aggrDataPoints[i].hdd));
- }
-
- // Draw line chart.
- chartOptions.title = 'HDD Usage (%)';
- hddChart.draw(hddView, chartOptions);
-}
-
-/**
- * Sort json array
- */
-function sortByKey(array, key) {
- return array.sort(function(a, b) {
- var x = a[key]; var y = b[key];
- return ((x < y) ? -1 : ((x > y) ? 1 : 0));
- });
-}
-
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/machinedata/src/main/html/index.php
----------------------------------------------------------------------
diff --git a/demos/machinedata/src/main/html/index.php b/demos/machinedata/src/main/html/index.php
deleted file mode 100644
index 6b93570..0000000
--- a/demos/machinedata/src/main/html/index.php
+++ /dev/null
@@ -1,263 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
-<html>
-<head>
-<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
-<title>Machine Generated Data Demo </title>
-
-<link rel="stylesheet" type="text/css" href="malhar.css">
-
-<!-- Google charts include -->
-<script type="text/javascript" src="https://www.google.com/jsapi"></script>
-<script type="text/javascript">
-google.load('visualization', '1', {'packages':['corechart']});
-</script>
-
-<!-- Malhar charting utils -->
-<script type="text/javascript" src="global.js"></script>
-
-<!-- window onload -->
-<script type="text/javascript">
-
-function DrawAggrCharts()
-{
- // get refresh url
- lookback = aggrLookBack;
- var url = DataUrl();
-
- // fetch data, draw charts
- try
- {
- var connect = new XMLHttpRequest();
- connect.onreadystatechange = function() {
- if(connect.readyState==4 && connect.status==200) {
-
-console.log(url);
- aggrData = connect.response;
- var pts = JSON.parse(aggrData);
- aggrDataPoints = new Array();
- for(var i=0; i < pts.length; i++) aggrDataPoints.push(pts[i]);
- DrawCPUChart();
- DrawRAMChart();
- DrawHDDChart();
- //DrawImpressionsChart();
- delete aggrData;
- }
- }
- connect.open('GET', url, true);
- connect.send(null);
- } catch(e) {
- }
- aggrLookBack += 30;
-}
-
-function DrawContCharts()
-{
- // get refresh url
- lookback = contLookBack;
- var url = DataUrl();
- //document.getElementById('chart_div').innerHTML = url;
-
- // fetch data, draw charts
- try
- {
- var connect = new XMLHttpRequest();
- connect.onreadystatechange = function() {
- if(connect.readyState==4 && connect.status==200) {
- contData = connect.response;
- var newPts = JSON.parse(contData);
- contDataPoints = new Array();
- for(var i=0; i < newPts.length; i++) contDataPoints.push(newPts[i]);
- DrawCtrChart() ;
- DrawMarginChart();
- delete contData;
- delete newPts;
- }
- }
- connect.open('GET', url, true);
- connect.send(null);
- } catch(e) {
- }
- contLookBack += contRefresh;
-}
-
-window.onload = function() {
-
- // Initialize global
- InitializeGlobal();
-
- // Inituialize form fields
- if (params['customer']) document.getElementById('customer').value = params['customer'];
- if (params['product']) document.getElementById('product').value = params['product'];
- if (params['os']) document.getElementById('os').value = params['os'];
- if (params['software1']) document.getElementById('software1').value = params['software1'];
- if (params['software2']) document.getElementById('software2').value = params['software2'];
- if (params['software3']) document.getElementById('software3').value = params['software3'];
- if (params['refresh'])
- {
- document.getElementById('refresh').value = params['refresh'];
- } else {
- document.getElementById('refresh').value = 5;
- }
- if (params['lookback'])
- {
- document.getElementById('lookback').value = params['lookback'];
- } else {
- document.getElementById('lookback').value = 6;
- }
-
- // draw charts
- DrawAggrCharts();
- //DrawContCharts();
- setInterval(DrawAggrCharts, 30000);
- //setInterval(DrawContCharts, contRefresh * 1000);
-};
-
-</script>
-
-</head>
-<body>
-
- <div id="header">
- <ul class="dashboard-modes">
- <li>
- <a href="#" class="active">Machine Generated Data Demo </a>
- </li>
- </ul>
-
- </div>
-
- <div id="main">
- <div id="pagecontent">
- <div class="dashboardMgr">
- <div class="inner" style="">
- <h2 class="title">View Real Time Data Charts</h2>
- <form method="GET" action="index.php">
-
- <label for="customer">Customer ID:</label>
- <select name="customer" id="customer" style="width:200px;">
- <option value="">ALL</option>
- <?php
- for ($i = 1; $i <= 5; $i++) {
- print "<option value=\"$i\">Customer $i</option>\n";
- }
- ?>
- </select>
-
- <label for="">Product ID:</label>
- <select name="product" id="product" style="width:200px;">
- <option value="">ALL</option>
- <?php
- for ($i = 4; $i <= 6; $i++) {
- print "<option value=\"$i\">Product $i</option>\n";
- }
- ?>
- </select>
-
- <label for="">Product OS:</label>
- <select name="os" id="os" style="width:200px;">
- <option value="">ALL</option>
- <?php
- for ($i = 10; $i <= 12; $i++) {
- print "<option value=\"$i\">OS $i</option>\n";
- }
- ?>
- </select>
-
- <label for="software1">Software1 Ver:</label>
- <select name="software1" id="software1" style="width:200px;">
- <option value="">ALL</option>
- <?php
- for ($i = 10; $i <= 12; $i++) {
- print "<option value=\"$i\">Software1 Version $i</option>\n";
- }
- ?>
- </select>
-
- <label for="software2">Software2 Ver:</label>
- <select name="software2" id="software2" style="width:200px;">
- <option value="">ALL</option>
- <?php
- for ($i = 12; $i <= 14; $i++) {
- print "<option value=\"$i\">Software2 Version $i</option>\n";
- }
- ?>
- </select>
-
- <label for="software3">Software3 Ver:</label>
- <select name="software3" id="software3" style="width:200px;">
- <option value="">ALL</option>
- <?php
- for ($i = 4; $i <= 6; $i++) {
- print "<option value=\"$i\">Software3 Version $i</option>\n";
- }
- ?>
- </select>
-
- <label for="">Refresh Interval:</label>
- <div class="input-append">
- <input type="text" name="refresh" id="refresh" class="input-small"/>
- <span class="add-on">Secs</span>
- </div>
-
-
- <label for="">Look Back:</label>
- <div class="input-append">
- <input type="text" name="lookback" id="lookback" class="input-small"/>
- <span class="add-on">Hours</span>
- </div>
-
- <input type="submit" value="submit" class="btn btn-primary" />
-
- </form>
- </div>
- <div class="collapser-container">
- <div class="collapser">
- <div class="collapse-dot"></div>
- <div class="collapse-dot"></div>
- <div class="collapse-dot"></div>
- </div>
- </div>
- </div>
- <div class="dashboardMain">
-
- <!-- <table><tbody>
- <tr>
- <td><div id="chart_div"></div></td>
- <td><div id="chart1_div" ></div></td>
- </tr>
- <tr>
- <td><div id="chart2_div" ></div></td>
- <td><div id="chart3_div" ></div></td>
- </tr>
- <tr>
- <td><div id="chart4_div" ></div></td>
- <td><div id="chart5_div" ></div></td>
- </tr>
- </tr></tbody></table> -->
- <div class="chart-ctnr" id="chart_div"></div>
- <div class="chart-ctnr" id="chart1_div" ></div>
- <div class="chart-ctnr" id="chart2_div" ></div>
-<!-- <div class="chart-ctnr" id="chart3_div" ></div>
- <div class="chart-ctnr" id="chart4_div" ></div>
- <div class="chart-ctnr" id="chart5_div" ></div> -->
- </div>
-</body>
-</html>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/machinedata/src/main/html/json.php
----------------------------------------------------------------------
diff --git a/demos/machinedata/src/main/html/json.php b/demos/machinedata/src/main/html/json.php
deleted file mode 100644
index 75a7117..0000000
--- a/demos/machinedata/src/main/html/json.php
+++ /dev/null
@@ -1,96 +0,0 @@
-<?php
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-header("Content-type: application/json");
-$redis = new Redis();
-$redis->connect('localhost');
-$redis->select(15);
-$from = $_GET['from'];
-$bucket = $_GET['bucket'];
-$customer = $_GET['customer'];
-$product = $_GET['product'];
-$os = $_GET['os'];
-$software1 = $_GET['software1'];
-$software2 = $_GET['software2'];
-$software3 = $_GET['software3'];
-
-switch ($bucket) {
-case 'D':
- $format = 'Ymd';
- $incr = 60 * 60 * 24;
- break;
-case 'h':
- $format = 'YmdH';
- $incr = 60 * 60;
- break;
-case 'm':
- $format = 'YmdHi';
- $incr = 60;
- break;
-default:
- break;
-}
-
-$arr = array();
-if ($customer != '') {
- $arr[] = "0:".$customer;
-}
-if ($product != '') {
- $arr[] = "1:".$product;
-}
-if ($os != '') {
- $arr[] = "2:".$os;
-}
-if ($software1 != '') {
- $arr[] = "3:".$software1;
-}
-if ($software2 != '') {
- $arr[] = "4:".$software2;
-}
-if ($software3 != '') {
- $arr[] = "5:".$software3;
-}
-$subpattern = "";
-if (count($arr) != 0) {
- $subpattern = join("|", $arr);
-}
-
-$result = array();
-
-while ($from < time()) {
- $date = gmdate($format, $from);
- if ($subpattern != '') {
- $key = $bucket . '|' . $date . '|' . $subpattern;
- } else {
- $key = $bucket . '|' . $date ;
- }
- $hash = $redis->hGetAll($key);
- if ($hash) {
- $cpu = $hash['cpu'];
- $ram = $hash['ram'];
- $hdd = $hash['hdd'];
- $result[] = array('timestamp'=> $from * 1000, 'cpu'=>$cpu, 'ram'=>$ram, 'hdd'=>$hdd);
- }
- $from += $incr;
-}
-
-array_pop($result);
-print json_encode($result);
-
-?>
[11/30] apex-malhar git commit: Renamed demos to examples. Packages
and artifactid names are changed as suggested.
Posted by th...@apache.org.
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/cookbook/MaxPerKeyExamples.java
----------------------------------------------------------------------
diff --git a/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/cookbook/MaxPerKeyExamples.java b/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/cookbook/MaxPerKeyExamples.java
new file mode 100644
index 0000000..9fd9495
--- /dev/null
+++ b/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/cookbook/MaxPerKeyExamples.java
@@ -0,0 +1,205 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.malhar.stream.sample.cookbook;
+
+import java.util.List;
+
+import org.apache.apex.malhar.lib.window.TriggerOption;
+import org.apache.apex.malhar.lib.window.Tuple;
+import org.apache.apex.malhar.lib.window.Window;
+import org.apache.apex.malhar.lib.window.WindowOption;
+import org.apache.apex.malhar.lib.window.accumulation.Max;
+import org.apache.apex.malhar.stream.api.ApexStream;
+import org.apache.apex.malhar.stream.api.CompositeStreamTransform;
+import org.apache.apex.malhar.stream.api.WindowedStream;
+import org.apache.apex.malhar.stream.api.function.Function;
+import org.apache.apex.malhar.stream.api.impl.StreamFactory;
+import org.apache.hadoop.conf.Configuration;
+
+import static java.sql.Types.DOUBLE;
+import static java.sql.Types.INTEGER;
+
+import com.google.common.collect.Lists;
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.lib.db.jdbc.JdbcFieldInfo;
+import com.datatorrent.lib.db.jdbc.JdbcPOJOInputOperator;
+import com.datatorrent.lib.db.jdbc.JdbcPOJOInsertOutputOperator;
+import com.datatorrent.lib.db.jdbc.JdbcStore;
+import com.datatorrent.lib.db.jdbc.JdbcTransactionalStore;
+import com.datatorrent.lib.util.FieldInfo;
+import com.datatorrent.lib.util.KeyValPair;
+
+import static org.apache.apex.malhar.stream.api.Option.Options.name;
+
+/**
+ * MaxPerKeyExamples Application from Beam
+ *
+ * @since 3.5.0
+ */
+@ApplicationAnnotation(name = "MaxPerKeyExamples")
+public class MaxPerKeyExamples implements StreamingApplication
+{
+
+ /**
+ * A map function to extract the mean temperature from {@link InputPojo}.
+ */
+ public static class ExtractTempFn implements Function.MapFunction<InputPojo, KeyValPair<Integer, Double>>
+ {
+ @Override
+ public KeyValPair<Integer, Double> f(InputPojo row)
+ {
+ Integer month = row.getMonth();
+ Double meanTemp = row.getMeanTemp();
+ return new KeyValPair<Integer, Double>(month, meanTemp);
+ }
+ }
+
+
+ /**
+ * A map function to format output to {@link OutputPojo}.
+ */
+ public static class FormatMaxesFn implements Function.MapFunction<Tuple.WindowedTuple<KeyValPair<Integer, Double>>, OutputPojo>
+ {
+ @Override
+ public OutputPojo f(Tuple.WindowedTuple<KeyValPair<Integer, Double>> input)
+ {
+ OutputPojo row = new OutputPojo();
+ row.setMonth(input.getValue().getKey());
+ row.setMeanTemp(input.getValue().getValue());
+ return row;
+ }
+ }
+
+ /**
+ * A composite transformation to perform three tasks:
+ * 1. extract the month and its mean temperature from input pojo.
+ * 2. find the maximum mean temperature for every month.
+ * 3. format the result to a output pojo object.
+ */
+ public static class MaxMeanTemp extends CompositeStreamTransform<WindowedStream<InputPojo>, WindowedStream<OutputPojo>>
+ {
+ @Override
+ public WindowedStream<OutputPojo> compose(WindowedStream<InputPojo> rows)
+ {
+ // InputPojo... => <month, meanTemp> ...
+ WindowedStream<KeyValPair<Integer, Double>> temps = rows.map(new ExtractTempFn(), name("ExtractTempFn"));
+
+ // month, meanTemp... => <month, max mean temp>...
+ WindowedStream<Tuple.WindowedTuple<KeyValPair<Integer, Double>>> tempMaxes =
+ temps.accumulateByKey(new Max<Double>(),
+ new Function.ToKeyValue<KeyValPair<Integer, Double>, Integer, Double>()
+ {
+ @Override
+ public Tuple<KeyValPair<Integer, Double>> f(KeyValPair<Integer, Double> input)
+ {
+ return new Tuple.WindowedTuple<KeyValPair<Integer, Double>>(Window.GlobalWindow.INSTANCE, input);
+ }
+ }, name("MaxPerMonth"));
+
+ // <month, max>... => OutputPojo...
+ WindowedStream<OutputPojo> results = tempMaxes.map(new FormatMaxesFn(), name("FormatMaxesFn"));
+
+ return results;
+ }
+ }
+
+ /**
+ * Method to set field info for {@link JdbcPOJOInputOperator}.
+ * @return
+ */
+ private List<FieldInfo> addInputFieldInfos()
+ {
+ List<FieldInfo> fieldInfos = Lists.newArrayList();
+ fieldInfos.add(new FieldInfo("MONTH", "month", FieldInfo.SupportType.INTEGER));
+ fieldInfos.add(new FieldInfo("DAY", "day", FieldInfo.SupportType.INTEGER));
+ fieldInfos.add(new FieldInfo("YEAR", "year", FieldInfo.SupportType.INTEGER));
+ fieldInfos.add(new FieldInfo("MEANTEMP", "meanTemp", FieldInfo.SupportType.DOUBLE));
+ return fieldInfos;
+ }
+
+ /**
+ * Method to set field info for {@link JdbcPOJOInsertOutputOperator}.
+ * @return
+ */
+ private List<JdbcFieldInfo> addOutputFieldInfos()
+ {
+ List<JdbcFieldInfo> fieldInfos = Lists.newArrayList();
+ fieldInfos.add(new JdbcFieldInfo("MONTH", "month", JdbcFieldInfo.SupportType.INTEGER, INTEGER));
+ fieldInfos.add(new JdbcFieldInfo("MEANTEMP", "meanTemp", JdbcFieldInfo.SupportType.DOUBLE, DOUBLE));
+ return fieldInfos;
+ }
+
+
+ /**
+ * Populate the dag using High-Level API.
+ * @param dag
+ * @param conf
+ */
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ JdbcPOJOInputOperator jdbcInput = new JdbcPOJOInputOperator();
+ jdbcInput.setFieldInfos(addInputFieldInfos());
+
+ JdbcStore store = new JdbcStore();
+ jdbcInput.setStore(store);
+
+ JdbcPOJOInsertOutputOperator jdbcOutput = new JdbcPOJOInsertOutputOperator();
+ jdbcOutput.setFieldInfos(addOutputFieldInfos());
+ JdbcTransactionalStore outputStore = new JdbcTransactionalStore();
+ jdbcOutput.setStore(outputStore);
+
+ // Create stream that reads from a Jdbc Input.
+ ApexStream<Object> stream = StreamFactory.fromInput(jdbcInput, jdbcInput.outputPort, name("jdbcInput"))
+
+ // Apply window and trigger option to the stream.
+ .window(new WindowOption.GlobalWindow(), new TriggerOption().accumulatingFiredPanes().withEarlyFiringsAtEvery(1))
+
+ // Because Jdbc Input sends out a stream of Object, need to cast them to InputPojo.
+ .map(new Function.MapFunction<Object, InputPojo>()
+ {
+ @Override
+ public InputPojo f(Object input)
+ {
+ return (InputPojo)input;
+ }
+ }, name("ObjectToInputPojo"))
+
+ // Plug in the composite transformation to the stream to calculate the maximum temperature for each month.
+ .addCompositeStreams(new MaxMeanTemp())
+
+ // Cast the resulted OutputPojo to Object for Jdbc Output to consume.
+ .map(new Function.MapFunction<OutputPojo, Object>()
+ {
+ @Override
+ public Object f(OutputPojo input)
+ {
+ return (Object)input;
+ }
+ }, name("OutputPojoToObject"))
+
+ // Output the result to Jdbc Output.
+ .endWith(jdbcOutput, jdbcOutput.input, name("jdbcOutput"));
+
+ stream.populateDag(dag);
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/cookbook/OutputPojo.java
----------------------------------------------------------------------
diff --git a/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/cookbook/OutputPojo.java b/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/cookbook/OutputPojo.java
new file mode 100644
index 0000000..f3d0c64
--- /dev/null
+++ b/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/cookbook/OutputPojo.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.malhar.stream.sample.cookbook;
+
+/**
+ * OutputPojo Tuple Class for jdbcOutput of {@link MaxPerKeyExamples}.
+ *
+ * @since 3.5.0
+ */
+public class OutputPojo
+{
+ private int month;
+ private double meanTemp;
+
+ @Override
+ public String toString()
+ {
+ return "PojoEvent [month=" + getMonth() + ", meanTemp=" + getMeanTemp() + "]";
+ }
+
+ public void setMonth(int month)
+ {
+ this.month = month;
+ }
+
+ public int getMonth()
+ {
+ return this.month;
+ }
+
+ public void setMeanTemp(double meanTemp)
+ {
+ this.meanTemp = meanTemp;
+ }
+
+ public double getMeanTemp()
+ {
+ return meanTemp;
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/cookbook/TriggerExample.java
----------------------------------------------------------------------
diff --git a/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/cookbook/TriggerExample.java b/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/cookbook/TriggerExample.java
new file mode 100644
index 0000000..962faa5
--- /dev/null
+++ b/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/cookbook/TriggerExample.java
@@ -0,0 +1,577 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.malhar.stream.sample.cookbook;
+
+import java.util.Date;
+import java.util.Objects;
+
+import org.joda.time.Duration;
+
+import org.apache.apex.malhar.lib.window.TriggerOption;
+import org.apache.apex.malhar.lib.window.Tuple;
+import org.apache.apex.malhar.lib.window.WindowOption;
+import org.apache.apex.malhar.stream.api.ApexStream;
+import org.apache.apex.malhar.stream.api.CompositeStreamTransform;
+import org.apache.apex.malhar.stream.api.WindowedStream;
+import org.apache.apex.malhar.stream.api.function.Function;
+import org.apache.apex.malhar.stream.api.impl.StreamFactory;
+
+import com.datatorrent.lib.util.KeyValPair;
+
+/**
+ * This example illustrates the basic concepts behind triggering. It shows how to use different
+ * trigger definitions to produce partial (speculative) results before all the data is processed and
+ * to control when updated results are produced for late data. The example performs a streaming
+ * analysis of the data coming in from PubSub and writes the results to BigQuery. It divides the
+ * data into {@link Window windows} to be processed, and demonstrates using various kinds of
+ * {@link org.apache.beam.sdk.transforms.windowing.Trigger triggers} to control when the results for
+ * each window are emitted.
+ *
+ * <p> This example uses a portion of real traffic data from San Diego freeways. It contains
+ * readings from sensor stations set up along each freeway. Each sensor reading includes a
+ * calculation of the 'total flow' across all lanes in that freeway direction.
+ *
+ * <p> Concepts:
+ * <pre>
+ * 1. The default triggering behavior
+ * 2. Late data with the default trigger
+ * 3. How to get speculative estimates
+ * 4. Combining late data and speculative estimates
+ * </pre>
+ *
+ * <p> Before running this example, it will be useful to familiarize yourself with Dataflow triggers
+ * and understand the concept of 'late data',
+ * See: <a href="https://cloud.google.com/dataflow/model/triggers">
+ * https://cloud.google.com/dataflow/model/triggers </a> and
+ * <a href="https://cloud.google.com/dataflow/model/windowing#Advanced">
+ * https://cloud.google.com/dataflow/model/windowing#Advanced </a>
+ *
+ * <p> The example pipeline reads data from a Pub/Sub topic. By default, running the example will
+ * also run an auxiliary pipeline to inject data from the default {@code --input} file to the
+ * {@code --pubsubTopic}. The auxiliary pipeline puts a timestamp on the injected data so that the
+ * example pipeline can operate on <i>event time</i> (rather than arrival time). The auxiliary
+ * pipeline also randomly simulates late data, by setting the timestamps of some of the data
+ * elements to be in the past. You may override the default {@code --input} with the file of your
+ * choosing or set {@code --input=""} which will disable the automatic Pub/Sub injection, and allow
+ * you to use a separate tool to publish to the given topic.
+ *
+ * <p> The example is configured to use the default Pub/Sub topic and the default BigQuery table
+ * from the example common package (there are no defaults for a general Dataflow pipeline).
+ * You can override them by using the {@code --pubsubTopic}, {@code --bigQueryDataset}, and
+ * {@code --bigQueryTable} options. If the Pub/Sub topic or the BigQuery table do not exist,
+ * the example will try to create them.
+ *
+ * <p> The pipeline outputs its results to a BigQuery table.
+ * Here are some queries you can use to see interesting results:
+ * Replace {@code <enter_table_name>} in the query below with the name of the BigQuery table.
+ * Replace {@code <enter_window_interval>} in the query below with the window interval.
+ *
+ * <p> To see the results of the default trigger,
+ * Note: When you start up your pipeline, you'll initially see results from 'late' data. Wait after
+ * the window duration, until the first pane of non-late data has been emitted, to see more
+ * interesting results.
+ * {@code SELECT * FROM enter_table_name WHERE triggerType = "default" ORDER BY window DESC}
+ *
+ * <p> To see the late data i.e. dropped by the default trigger,
+ * {@code SELECT * FROM <enter_table_name> WHERE triggerType = "withAllowedLateness" and
+ * (timing = "LATE" or timing = "ON_TIME") and freeway = "5" ORDER BY window DESC, processingTime}
+ *
+ * <p>To see the the difference between accumulation mode and discarding mode,
+ * {@code SELECT * FROM <enter_table_name> WHERE (timing = "LATE" or timing = "ON_TIME") AND
+ * (triggerType = "withAllowedLateness" or triggerType = "sequential") and freeway = "5" ORDER BY
+ * window DESC, processingTime}
+ *
+ * <p> To see speculative results every minute,
+ * {@code SELECT * FROM <enter_table_name> WHERE triggerType = "speculative" and freeway = "5"
+ * ORDER BY window DESC, processingTime}
+ *
+ * <p> To see speculative results every five minutes after the end of the window
+ * {@code SELECT * FROM <enter_table_name> WHERE triggerType = "sequential" and timing != "EARLY"
+ * and freeway = "5" ORDER BY window DESC, processingTime}
+ *
+ * <p> To see the first and the last pane for a freeway in a window for all the trigger types,
+ * {@code SELECT * FROM <enter_table_name> WHERE (isFirst = true or isLast = true) ORDER BY window}
+ *
+ * <p> To reduce the number of results for each query we can add additional where clauses.
+ * For examples, To see the results of the default trigger,
+ * {@code SELECT * FROM <enter_table_name> WHERE triggerType = "default" AND freeway = "5" AND
+ * window = "<enter_window_interval>"}
+ *
+ * <p> The example will try to cancel the pipelines on the signal to terminate the process (CTRL-C)
+ * and then exits.
+ *
+ * @since 3.5.0
+ */
+
+public class TriggerExample
+{
+ //Numeric value of fixed window duration, in minutes
+ public static final int WINDOW_DURATION = 30;
+ // Constants used in triggers.
+ // Speeding up ONE_MINUTE or FIVE_MINUTES helps you get an early approximation of results.
+ // ONE_MINUTE is used only with processing time before the end of the window
+ public static final Duration ONE_MINUTE = Duration.standardMinutes(1);
+ // FIVE_MINUTES is used only with processing time after the end of the window
+ public static final Duration FIVE_MINUTES = Duration.standardMinutes(5);
+ // ONE_DAY is used to specify the amount of lateness allowed for the data elements.
+ public static final Duration ONE_DAY = Duration.standardDays(1);
+
+ /**
+ * This transform demonstrates using triggers to control when data is produced for each window
+ * Consider an example to understand the results generated by each type of trigger.
+ * The example uses "freeway" as the key. Event time is the timestamp associated with the data
+ * element and processing time is the time when the data element gets processed in the pipeline.
+ * For freeway 5, suppose there are 10 elements in the [10:00:00, 10:30:00) window.
+ * Key (freeway) | Value (totalFlow) | event time | processing time
+ * 5 | 50 | 10:00:03 | 10:00:47
+ * 5 | 30 | 10:01:00 | 10:01:03
+ * 5 | 30 | 10:02:00 | 11:07:00
+ * 5 | 20 | 10:04:10 | 10:05:15
+ * 5 | 60 | 10:05:00 | 11:03:00
+ * 5 | 20 | 10:05:01 | 11.07:30
+ * 5 | 60 | 10:15:00 | 10:27:15
+ * 5 | 40 | 10:26:40 | 10:26:43
+ * 5 | 60 | 10:27:20 | 10:27:25
+ * 5 | 60 | 10:29:00 | 11:11:00
+ *
+ * <p> Dataflow tracks a watermark which records up to what point in event time the data is
+ * complete. For the purposes of the example, we'll assume the watermark is approximately 15m
+ * behind the current processing time. In practice, the actual value would vary over time based
+ * on the systems knowledge of the current PubSub delay and contents of the backlog (data
+ * that has not yet been processed).
+ *
+ * <p> If the watermark is 15m behind, then the window [10:00:00, 10:30:00) (in event time) would
+ * close at 10:44:59, when the watermark passes 10:30:00.
+ */
+ static class CalculateTotalFlow
+ extends CompositeStreamTransform<ApexStream<String>, WindowedStream<SampleBean>>
+ {
+ private int windowDuration;
+
+ CalculateTotalFlow(int windowDuration)
+ {
+ this.windowDuration = windowDuration;
+ }
+
+ @Override
+ public WindowedStream<SampleBean> compose(ApexStream<String> inputStream)
+ {
+ // Concept #1: The default triggering behavior
+ // By default Dataflow uses a trigger which fires when the watermark has passed the end of the
+ // window. This would be written {@code Repeatedly.forever(AfterWatermark.pastEndOfWindow())}.
+
+ // The system also defaults to dropping late data -- data which arrives after the watermark
+ // has passed the event timestamp of the arriving element. This means that the default trigger
+ // will only fire once.
+
+ // Each pane produced by the default trigger with no allowed lateness will be the first and
+ // last pane in the window, and will be ON_TIME.
+
+ // The results for the example above with the default trigger and zero allowed lateness
+ // would be:
+ // Key (freeway) | Value (totalFlow) | numberOfRecords | isFirst | isLast | timing
+ // 5 | 260 | 6 | true | true | ON_TIME
+
+ // At 11:03:00 (processing time) the system watermark may have advanced to 10:54:00. As a
+ // result, when the data record with event time 10:05:00 arrives at 11:03:00, it is considered
+ // late, and dropped.
+
+ WindowedStream<SampleBean> defaultTriggerResults = inputStream
+ .window(new WindowOption.TimeWindows(Duration.standardMinutes(windowDuration)),
+ new TriggerOption().discardingFiredPanes())
+ .addCompositeStreams(new TotalFlow("default"));
+
+ // Concept #2: Late data with the default trigger
+ // This uses the same trigger as concept #1, but allows data that is up to ONE_DAY late. This
+ // leads to each window staying open for ONE_DAY after the watermark has passed the end of the
+ // window. Any late data will result in an additional pane being fired for that same window.
+
+ // The first pane produced will be ON_TIME and the remaining panes will be LATE.
+ // To definitely get the last pane when the window closes, use
+ // .withAllowedLateness(ONE_DAY, ClosingBehavior.FIRE_ALWAYS).
+
+ // The results for the example above with the default trigger and ONE_DAY allowed lateness
+ // would be:
+ // Key (freeway) | Value (totalFlow) | numberOfRecords | isFirst | isLast | timing
+ // 5 | 260 | 6 | true | false | ON_TIME
+ // 5 | 60 | 1 | false | false | LATE
+ // 5 | 30 | 1 | false | false | LATE
+ // 5 | 20 | 1 | false | false | LATE
+ // 5 | 60 | 1 | false | false | LATE
+ WindowedStream<SampleBean> withAllowedLatenessResults = inputStream
+ .window(new WindowOption.TimeWindows(Duration.standardMinutes(windowDuration)),
+ new TriggerOption().discardingFiredPanes(),
+ Duration.standardDays(1))
+ .addCompositeStreams(new TotalFlow("withAllowedLateness"));
+
+ // Concept #3: How to get speculative estimates
+ // We can specify a trigger that fires independent of the watermark, for instance after
+ // ONE_MINUTE of processing time. This allows us to produce speculative estimates before
+ // all the data is available. Since we don't have any triggers that depend on the watermark
+ // we don't get an ON_TIME firing. Instead, all panes are either EARLY or LATE.
+
+ // We also use accumulatingFiredPanes to build up the results across each pane firing.
+
+ // The results for the example above for this trigger would be:
+ // Key (freeway) | Value (totalFlow) | numberOfRecords | isFirst | isLast | timing
+ // 5 | 80 | 2 | true | false | EARLY
+ // 5 | 100 | 3 | false | false | EARLY
+ // 5 | 260 | 6 | false | false | EARLY
+ // 5 | 320 | 7 | false | false | LATE
+ // 5 | 370 | 9 | false | false | LATE
+ // 5 | 430 | 10 | false | false | LATE
+
+ ApexStream<SampleBean> speculativeResults = inputStream
+ .window(new WindowOption.TimeWindows(Duration.standardMinutes(windowDuration)),
+ //Trigger fires every minute
+ new TriggerOption().withEarlyFiringsAtEvery(Duration.standardMinutes(1))
+ // After emitting each pane, it will continue accumulating the elements so that each
+ // approximation includes all of the previous data in addition to the newly arrived
+ // data.
+ .accumulatingFiredPanes(),
+ Duration.standardDays(1))
+ .addCompositeStreams(new TotalFlow("speculative"));
+
+ // Concept #4: Combining late data and speculative estimates
+ // We can put the previous concepts together to get EARLY estimates, an ON_TIME result,
+ // and LATE updates based on late data.
+
+ // Each time a triggering condition is satisfied it advances to the next trigger.
+ // If there are new elements this trigger emits a window under following condition:
+ // > Early approximations every minute till the end of the window.
+ // > An on-time firing when the watermark has passed the end of the window
+ // > Every five minutes of late data.
+
+ // Every pane produced will either be EARLY, ON_TIME or LATE.
+
+ // The results for the example above for this trigger would be:
+ // Key (freeway) | Value (totalFlow) | numberOfRecords | isFirst | isLast | timing
+ // 5 | 80 | 2 | true | false | EARLY
+ // 5 | 100 | 3 | false | false | EARLY
+ // 5 | 260 | 6 | false | false | EARLY
+ // [First pane fired after the end of the window]
+ // 5 | 320 | 7 | false | false | ON_TIME
+ // 5 | 430 | 10 | false | false | LATE
+
+ // For more possibilities of how to build advanced triggers, see {@link Trigger}.
+ WindowedStream<SampleBean> sequentialResults = inputStream
+ .window(new WindowOption.TimeWindows(Duration.standardMinutes(windowDuration)),
+ // Speculative every ONE_MINUTE
+ new TriggerOption().withEarlyFiringsAtEvery(Duration.standardMinutes(1))
+ .withLateFiringsAtEvery(Duration.standardMinutes(5))
+ // After emitting each pane, it will continue accumulating the elements so that each
+ // approximation includes all of the previous data in addition to the newly arrived
+ // data.
+ .accumulatingFiredPanes(),
+ Duration.standardDays(1))
+ .addCompositeStreams(new TotalFlow("sequential"));
+
+ return sequentialResults;
+ }
+
+ }
+
+ //////////////////////////////////////////////////////////////////////////////////////////////////
+ // The remaining parts of the pipeline are needed to produce the output for each
+ // concept above. Not directly relevant to understanding the trigger examples.
+
+ /**
+ * Calculate total flow and number of records for each freeway and format the results to TableRow
+ * objects, to save to BigQuery.
+ */
+ static class TotalFlow extends
+ CompositeStreamTransform<WindowedStream<String>, WindowedStream<SampleBean>>
+ {
+ private String triggerType;
+
+ public TotalFlow(String triggerType)
+ {
+ this.triggerType = triggerType;
+ }
+
+ @Override
+ public WindowedStream<SampleBean> compose(WindowedStream<String> inputStream)
+ {
+
+ WindowedStream<KeyValPair<String, Iterable<Integer>>> flowPerFreeway = inputStream
+ .groupByKey(new ExtractFlowInfo());
+
+ return flowPerFreeway
+ .map(new Function.MapFunction<KeyValPair<String, Iterable<Integer>>, KeyValPair<String, String>>()
+ {
+ @Override
+ public KeyValPair<String, String> f(KeyValPair<String, Iterable<Integer>> input)
+ {
+ Iterable<Integer> flows = input.getValue();
+ Integer sum = 0;
+ Long numberOfRecords = 0L;
+ for (Integer value : flows) {
+ sum += value;
+ numberOfRecords++;
+ }
+ return new KeyValPair<>(input.getKey(), sum + "," + numberOfRecords);
+ }
+ })
+ .map(new FormatTotalFlow(triggerType));
+ }
+ }
+
+ /**
+ * Format the results of the Total flow calculation to a TableRow, to save to BigQuery.
+ * Adds the triggerType, pane information, processing time and the window timestamp.
+ */
+ static class FormatTotalFlow implements Function.MapFunction<KeyValPair<String, String>, SampleBean>
+ {
+ private String triggerType;
+
+ public FormatTotalFlow(String triggerType)
+ {
+ this.triggerType = triggerType;
+ }
+
+ @Override
+ public SampleBean f(KeyValPair<String, String> input)
+ {
+ String[] values = input.getValue().split(",");
+ //TODO need to have a callback to get the metadata like window id, pane id, timestamps etc.
+ return new SampleBean(triggerType, input.getKey(), Integer.parseInt(values[0]), Long
+ .parseLong(values[1]), null, false, false, null, null, new Date());
+ }
+ }
+
+ public static class SampleBean
+ {
+ public SampleBean()
+ {
+ }
+
+ private String triggerType;
+
+ private String freeway;
+
+ private int totalFlow;
+
+ private long numberOfRecords;
+
+ private String window;
+
+ private boolean isFirst;
+
+ private boolean isLast;
+
+ private Date timing;
+
+ private Date eventTime;
+
+ private Date processingTime;
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ SampleBean that = (SampleBean)o;
+ return totalFlow == that.totalFlow &&
+ numberOfRecords == that.numberOfRecords &&
+ isFirst == that.isFirst &&
+ isLast == that.isLast &&
+ Objects.equals(triggerType, that.triggerType) &&
+ Objects.equals(freeway, that.freeway) &&
+ Objects.equals(window, that.window) &&
+ Objects.equals(timing, that.timing) &&
+ Objects.equals(eventTime, that.eventTime) &&
+ Objects.equals(processingTime, that.processingTime);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects
+ .hash(triggerType, freeway, totalFlow, numberOfRecords, window, isFirst, isLast, timing, eventTime,
+ processingTime);
+ }
+
+ public SampleBean(String triggerType, String freeway, int totalFlow, long numberOfRecords, String window, boolean isFirst, boolean isLast, Date timing, Date eventTime, Date processingTime)
+ {
+
+ this.triggerType = triggerType;
+ this.freeway = freeway;
+ this.totalFlow = totalFlow;
+ this.numberOfRecords = numberOfRecords;
+ this.window = window;
+ this.isFirst = isFirst;
+ this.isLast = isLast;
+ this.timing = timing;
+ this.eventTime = eventTime;
+ this.processingTime = processingTime;
+ }
+
+ public String getTriggerType()
+ {
+ return triggerType;
+ }
+
+ public void setTriggerType(String triggerType)
+ {
+ this.triggerType = triggerType;
+ }
+
+ public String getFreeway()
+ {
+ return freeway;
+ }
+
+ public void setFreeway(String freeway)
+ {
+ this.freeway = freeway;
+ }
+
+ public int getTotalFlow()
+ {
+ return totalFlow;
+ }
+
+ public void setTotalFlow(int totalFlow)
+ {
+ this.totalFlow = totalFlow;
+ }
+
+ public long getNumberOfRecords()
+ {
+ return numberOfRecords;
+ }
+
+ public void setNumberOfRecords(long numberOfRecords)
+ {
+ this.numberOfRecords = numberOfRecords;
+ }
+
+ public String getWindow()
+ {
+ return window;
+ }
+
+ public void setWindow(String window)
+ {
+ this.window = window;
+ }
+
+ public boolean isFirst()
+ {
+ return isFirst;
+ }
+
+ public void setFirst(boolean first)
+ {
+ isFirst = first;
+ }
+
+ public boolean isLast()
+ {
+ return isLast;
+ }
+
+ public void setLast(boolean last)
+ {
+ isLast = last;
+ }
+
+ public Date getTiming()
+ {
+ return timing;
+ }
+
+ public void setTiming(Date timing)
+ {
+ this.timing = timing;
+ }
+
+ public Date getEventTime()
+ {
+ return eventTime;
+ }
+
+ public void setEventTime(Date eventTime)
+ {
+ this.eventTime = eventTime;
+ }
+
+ public Date getProcessingTime()
+ {
+ return processingTime;
+ }
+
+ public void setProcessingTime(Date processingTime)
+ {
+ this.processingTime = processingTime;
+ }
+ }
+
+ /**
+ * Extract the freeway and total flow in a reading.
+ * Freeway is used as key since we are calculating the total flow for each freeway.
+ */
+ static class ExtractFlowInfo implements Function.ToKeyValue<String, String, Integer>
+ {
+ @Override
+ public Tuple<KeyValPair<String, Integer>> f(String input)
+ {
+ String[] laneInfo = input.split(",");
+ if (laneInfo[0].equals("timestamp")) {
+ // Header row
+ return null;
+ }
+ if (laneInfo.length < 48) {
+ //Skip the invalid input.
+ return null;
+ }
+ String freeway = laneInfo[2];
+ Integer totalFlow = tryIntegerParse(laneInfo[7]);
+ // Ignore the records with total flow 0 to easily understand the working of triggers.
+ // Skip the records with total flow -1 since they are invalid input.
+ if (totalFlow == null || totalFlow <= 0) {
+ return null;
+ }
+ return new Tuple.PlainTuple<>(new KeyValPair<>(freeway, totalFlow));
+ }
+ }
+
+ private static final String PUBSUB_TIMESTAMP_LABEL_KEY = "timestamp_ms";
+
+ public static void main(String[] args) throws Exception
+ {
+ StreamFactory.fromFolder("some folder")
+ .addCompositeStreams(new CalculateTotalFlow(60));
+
+ }
+
+ private static Integer tryIntegerParse(String number)
+ {
+ try {
+ return Integer.parseInt(number);
+ } catch (NumberFormatException e) {
+ return null;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/highlevelapi/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/examples/highlevelapi/src/main/resources/META-INF/properties.xml b/examples/highlevelapi/src/main/resources/META-INF/properties.xml
new file mode 100644
index 0000000..ead0460
--- /dev/null
+++ b/examples/highlevelapi/src/main/resources/META-INF/properties.xml
@@ -0,0 +1,141 @@
+<?xml version="1.0"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<configuration>
+ <!--
+ <property>
+ <name>dt.application.{appName}.operator.{opName}.prop.{propName}</name>
+ <value>some-default-value (if value is not specified, it is required from the user or custom config when launching)</value>
+ </property>
+ -->
+
+ <!-- Properties for TwitterAutoComplete, please fill out all of them to make the application work -->
+ <property>
+ <name>dt.application.TwitterAutoComplete.operator.tweetSampler.consumerKey</name>
+ <value></value>
+ </property>
+ <property>
+ <name>dt.application.TwitterAutoComplete.operator.tweetSampler.consumerSecret</name>
+ <value></value>
+ </property>
+ <property>
+ <name>dt.application.TwitterAutoComplete.operator.tweetSampler.accessToken</name>
+ <value></value>
+ </property>
+ <property>
+ <name>dt.application.TwitterAutoComplete.operator.tweetSampler.accessTokenSecret</name>
+ <value></value>
+ </property>
+
+ <!-- Properties for StreamingWordExtract -->
+ <property>
+ <name>dt.application.StreamingWordExtract.operator.jdbcOutput.prop.store.userName</name>
+ <value>root</value>
+ </property>
+ <property>
+ <name>dt.application.StreamingWordExtract.operator.jdbcOutput.prop.store.password</name>
+ <value>password</value>
+ </property>
+ <property>
+ <name>dt.application.StreamingWordExtract.operator.jdbcOutput.prop.store.databaseDriver</name>
+ <value>org.hsqldb.jdbcDriver</value>
+ </property>
+ <property>
+ <name>dt.application.StreamingWordExtract.operator.jdbcOutput.prop.batchSize</name>
+ <value>5</value>
+ </property>
+ <property>
+ <name>dt.application.StreamingWordExtract.operator.jdbcOutput.port.input.attr.TUPLE_CLASS</name>
+ <value>org.apache.apex.malhar.stream.sample.complete.PojoEvent</value>
+ </property>
+ <property>
+ <name>dt.application.StreamingWordExtract.operator.jdbcOutput.prop.store.databaseUrl</name>
+ <value>jdbc:hsqldb:mem:test</value>
+ </property>
+ <property>
+ <name>dt.application.StreamingWordExtract.operator.jdbcOutput.prop.tablename</name>
+ <value>Test</value>
+ </property>
+
+ <!-- Properties for MaxPerKeyExamples -->
+ <property>
+ <name>dt.application.MaxPerKeyExamples.operator.jdbcInput.prop.store.userName</name>
+ <value>root</value>
+ </property>
+ <property>
+ <name>dt.application.MaxPerKeyExamples.operator.jdbcInput.prop.store.password</name>
+ <value>password</value>
+ </property>
+ <property>
+ <name>dt.application.MaxPerKeyExamples.operator.jdbcInput.prop.store.databaseDriver</name>
+ <value>org.hsqldb.jdbcDriver</value>
+ </property>
+ <property>
+ <name>dt.application.MaxPerKeyExamples.operator.jdbcInput.prop.batchSize</name>
+ <value>5</value>
+ </property>
+ <property>
+ <name>dt.application.MaxPerKeyExamples.operator.jdbcInput.port.outputPort.attr.TUPLE_CLASS</name>
+ <value>org.apache.apex.malhar.stream.sample.cookbook.InputPojo</value>
+ </property>
+ <property>
+ <name>dt.application.MaxPerKeyExamples.operator.jdbcInput.prop.store.databaseUrl</name>
+ <value>jdbc:hsqldb:mem:test</value>
+ </property>
+ <property>
+ <name>dt.application.MaxPerKeyExamples.operator.jdbcInput.prop.tableName</name>
+ <value>InputTable</value>
+ </property>
+ <property>
+ <name>dt.application.MaxPerKeyExamples.operator.jdbcInput.prop.query</name>
+ <value>SELECT * FROM InputTable;</value>
+ </property>
+ <property>
+ <name>dt.application.MaxPerKeyExamples.operator.jdbcOutput.prop.store.userName</name>
+ <value>root</value>
+ </property>
+ <property>
+ <name>dt.application.MaxPerKeyExamples.operator.jdbcOutput.prop.store.password</name>
+ <value>password</value>
+ </property>
+ <property>
+ <name>dt.application.MaxPerKeyExamples.operator.jdbcOutput.prop.store.databaseDriver</name>
+ <value>org.hsqldb.jdbcDriver</value>
+ </property>
+ <property>
+ <name>dt.application.MaxPerKeyExamples.operator.jdbcOutput.prop.batchSize</name>
+ <value>5</value>
+ </property>
+ <property>
+ <name>dt.application.MaxPerKeyExamples.operator.jdbcOutput.port.input.attr.TUPLE_CLASS</name>
+ <value>org.apache.apex.malhar.stream.sample.cookbook.OutputPojo</value>
+ </property>
+ <property>
+ <name>dt.application.MaxPerKeyExamples.operator.jdbcOutput.prop.store.databaseUrl</name>
+ <value>jdbc:hsqldb:mem:test</value>
+ </property>
+ <property>
+ <name>dt.application.MaxPerKeyExamples.operator.jdbcOutput.prop.tablename</name>
+ <value>OutputTable</value>
+ </property>
+
+</configuration>
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/MinimalWordCountTest.java
----------------------------------------------------------------------
diff --git a/examples/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/MinimalWordCountTest.java b/examples/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/MinimalWordCountTest.java
new file mode 100644
index 0000000..c078683
--- /dev/null
+++ b/examples/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/MinimalWordCountTest.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.malhar.stream.sample;
+
+import java.util.concurrent.Callable;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.LocalMode;
+import com.datatorrent.stram.StramLocalCluster;
+
+/**
+ * Test for {@link MinimalWordCount}.
+ */
+public class MinimalWordCountTest
+{
+ @Test
+ public void MinimalWordCountTest() throws Exception
+ {
+ LocalMode lma = LocalMode.newInstance();
+ Configuration conf = new Configuration(false);
+ conf.set("dt.application.MinimalWordCount.operator.console.silent", "true");
+ MinimalWordCount app = new MinimalWordCount();
+
+ lma.prepareDAG(app, conf);
+
+ LocalMode.Controller lc = lma.getController();
+ ((StramLocalCluster)lc).setExitCondition(new Callable<Boolean>()
+ {
+ @Override
+ public Boolean call() throws Exception
+ {
+ return MinimalWordCount.Collector.isDone();
+ }
+ });
+
+ lc.run(10000);
+
+ Assert.assertTrue(MinimalWordCount.Collector.result.get("error") == 7);
+ Assert.assertTrue(MinimalWordCount.Collector.result.get("word") == 119);
+ Assert.assertTrue(MinimalWordCount.Collector.result.get("bye") == 1);
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/WindowedWordCountTest.java
----------------------------------------------------------------------
diff --git a/examples/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/WindowedWordCountTest.java b/examples/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/WindowedWordCountTest.java
new file mode 100644
index 0000000..f0c51f6
--- /dev/null
+++ b/examples/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/WindowedWordCountTest.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.malhar.stream.sample;
+
+import java.util.Map;
+import java.util.concurrent.Callable;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.LocalMode;
+import com.datatorrent.lib.util.KeyValPair;
+import com.datatorrent.stram.StramLocalCluster;
+
+/**
+ * Testing the TwitterAutoComplete Application. In order to run this test, you need to create an app
+ * at https://apps.twitter.com, then generate your consumer and access keys and tokens, and set the following properties
+ * for the application before running it:
+ * Your application consumer key,
+ * Your application consumer secret,
+ * Your twitter access token, and
+ * Your twitter access token secret.
+ */
+public class WindowedWordCountTest
+{
+ @Test
+ public void WindowedWordCountTest() throws Exception
+ {
+ LocalMode lma = LocalMode.newInstance();
+ Configuration conf = new Configuration(false);
+ conf.set("dt.application.WindowedWordCount.operator.console.silent", "true");
+ lma.prepareDAG(new WindowedWordCount(), conf);
+ LocalMode.Controller lc = lma.getController();
+ ((StramLocalCluster)lc).setExitCondition(new Callable<Boolean>()
+ {
+ @Override
+ public Boolean call() throws Exception
+ {
+ return WindowedWordCount.Collector.isDone();
+ }
+ });
+
+ lc.run(60000);
+
+ Assert.assertEquals(127, countSum(WindowedWordCount.Collector.getResult()));
+ Assert.assertEquals(28, countSumWord(WindowedWordCount.Collector.getResult(), "word2"));
+ Assert.assertEquals(7, countSumWord(WindowedWordCount.Collector.getResult(), "error"));
+ Assert.assertEquals(21, countSumWord(WindowedWordCount.Collector.getResult(), "word9"));
+ Assert.assertEquals(1, countSumWord(WindowedWordCount.Collector.getResult(), "bye"));
+ }
+
+ public long countSum(Map<KeyValPair<Long, String>, Long> map)
+ {
+ long sum = 0;
+ for (long count : map.values()) {
+ sum += count;
+ }
+ return sum;
+ }
+
+ public long countSumWord(Map<KeyValPair<Long, String>, Long> map, String word)
+ {
+ long sum = 0;
+ for (Map.Entry<KeyValPair<Long, String>, Long> entry : map.entrySet()) {
+ if (entry.getKey().getValue().equals(word)) {
+ sum += entry.getValue();
+ }
+ }
+ return sum;
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/complete/AutoCompleteTest.java
----------------------------------------------------------------------
diff --git a/examples/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/complete/AutoCompleteTest.java b/examples/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/complete/AutoCompleteTest.java
new file mode 100644
index 0000000..4ed2d5d
--- /dev/null
+++ b/examples/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/complete/AutoCompleteTest.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.malhar.stream.sample.complete;
+
+import java.util.concurrent.Callable;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.LocalMode;
+import com.datatorrent.stram.StramLocalCluster;
+
+/**
+ * Testing the AutoComplete Application
+ */
+public class AutoCompleteTest
+{
+
+ @Test
+ public void AutoCompleteTest() throws Exception
+ {
+ LocalMode lma = LocalMode.newInstance();
+ Configuration conf = new Configuration(false);
+ conf.set("dt.application.AutoComplete.operator.console.silent", "true");
+ lma.prepareDAG(new AutoComplete(), conf);
+ LocalMode.Controller lc = lma.getController();
+
+ ((StramLocalCluster)lc).setExitCondition(new Callable<Boolean>()
+ {
+ @Override
+ public Boolean call() throws Exception
+ {
+ return AutoComplete.Collector.isDone();
+ }
+ });
+
+ lc.run(200000);
+
+ Assert.assertTrue(AutoComplete.Collector.getResult().containsKey("had"));
+ Assert.assertTrue(AutoComplete.Collector.getResult().containsKey("hadoop"));
+ Assert.assertEquals(2, AutoComplete.Collector.getResult().get("mapreduce").get(0).getCount());
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/complete/StreamingWordExtractTest.java
----------------------------------------------------------------------
diff --git a/examples/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/complete/StreamingWordExtractTest.java b/examples/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/complete/StreamingWordExtractTest.java
new file mode 100644
index 0000000..dc9cdec
--- /dev/null
+++ b/examples/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/complete/StreamingWordExtractTest.java
@@ -0,0 +1,144 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.malhar.stream.sample.complete;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.concurrent.Callable;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.google.common.base.Throwables;
+import com.datatorrent.api.LocalMode;
+import com.datatorrent.lib.db.jdbc.JdbcTransactionalStore;
+import com.datatorrent.stram.StramLocalCluster;
+
+/**
+ * Testing StreamingWordExtract application
+ */
+public class StreamingWordExtractTest
+{
+ private static final String TUPLE_CLASS = "org.apache.apex.malhar.stream.sample.complete.PojoEvent";
+ private static final String DB_DRIVER = "org.h2.Driver";
+ private static final String DB_URL = "jdbc:h2:~/test";
+ private static final String TABLE_NAME = "Test";
+ private static final String USER_NAME = "root";
+ private static final String PSW = "password";
+
+ @BeforeClass
+ public static void setup()
+ {
+ try {
+ Class.forName(DB_DRIVER).newInstance();
+
+ Connection con = DriverManager.getConnection(DB_URL,USER_NAME,PSW);
+ Statement stmt = con.createStatement();
+
+ String createMetaTable = "CREATE TABLE IF NOT EXISTS " + JdbcTransactionalStore.DEFAULT_META_TABLE + " ( "
+ + JdbcTransactionalStore.DEFAULT_APP_ID_COL + " VARCHAR(100) NOT NULL, "
+ + JdbcTransactionalStore.DEFAULT_OPERATOR_ID_COL + " INT NOT NULL, "
+ + JdbcTransactionalStore.DEFAULT_WINDOW_COL + " BIGINT NOT NULL, "
+ + "UNIQUE (" + JdbcTransactionalStore.DEFAULT_APP_ID_COL + ", "
+ + JdbcTransactionalStore.DEFAULT_OPERATOR_ID_COL + ", " + JdbcTransactionalStore.DEFAULT_WINDOW_COL + ") "
+ + ")";
+ stmt.executeUpdate(createMetaTable);
+
+ String createTable = "CREATE TABLE IF NOT EXISTS " + TABLE_NAME
+ + "(STRINGVALUE VARCHAR(255))";
+ stmt.executeUpdate(createTable);
+
+ } catch (Throwable e) {
+ throw Throwables.propagate(e);
+ }
+ }
+
+ @After
+ public void cleanTable()
+ {
+ try {
+ Connection con = DriverManager.getConnection(DB_URL,USER_NAME,PSW);
+ Statement stmt = con.createStatement();
+ String dropTable = "drop table " + TABLE_NAME;
+ stmt.executeUpdate(dropTable);
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public void setConfig(Configuration conf)
+ {
+ conf.set("dt.operator.jdbcOutput.prop.store.userName", USER_NAME);
+ conf.set("dt.operator.jdbcOutput.prop.store.password", PSW);
+ conf.set("dt.operator.jdbcOutput.prop.store.databaseDriver", DB_DRIVER);
+ conf.set("dt.operator.jdbcOutput.prop.batchSize", "5");
+ conf.set("dt.operator.jdbcOutput.port.input.attr.TUPLE_CLASS", TUPLE_CLASS);
+ conf.set("dt.operator.jdbcOutput.prop.store.databaseUrl", DB_URL);
+ conf.set("dt.operator.jdbcOutput.prop.tablename", TABLE_NAME);
+ }
+
+ public int getNumOfEventsInStore()
+ {
+ Connection con;
+ try {
+ con = DriverManager.getConnection(DB_URL,USER_NAME,PSW);
+ Statement stmt = con.createStatement();
+
+ String countQuery = "SELECT count(*) from " + TABLE_NAME;
+ ResultSet resultSet = stmt.executeQuery(countQuery);
+ resultSet.next();
+ return resultSet.getInt(1);
+ } catch (SQLException e) {
+ throw new RuntimeException("fetching count", e);
+ }
+ }
+
+ @Test
+ public void StreamingWordExtractTest() throws Exception
+ {
+ LocalMode lma = LocalMode.newInstance();
+ Configuration conf = new Configuration(false);
+ setConfig(conf);
+ StreamingWordExtract app = new StreamingWordExtract();
+ lma.prepareDAG(app, conf);
+ LocalMode.Controller lc = lma.getController();
+
+ ((StramLocalCluster)lc).setExitCondition(new Callable<Boolean>()
+ {
+ @Override
+ public Boolean call() throws Exception
+ {
+ return getNumOfEventsInStore() == 36;
+ }
+ });
+
+ lc.run(10000);
+
+ Assert.assertEquals(app.getWordCount(), getNumOfEventsInStore());
+ Assert.assertEquals(app.getEntriesMapped(), getNumOfEventsInStore());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/complete/TopWikipediaSessionsTest.java
----------------------------------------------------------------------
diff --git a/examples/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/complete/TopWikipediaSessionsTest.java b/examples/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/complete/TopWikipediaSessionsTest.java
new file mode 100644
index 0000000..fddf511
--- /dev/null
+++ b/examples/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/complete/TopWikipediaSessionsTest.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.malhar.stream.sample.complete;
+
+import java.util.List;
+import java.util.concurrent.Callable;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.LocalMode;
+import com.datatorrent.stram.StramLocalCluster;
+
+/**
+ * Testing the {@link TopWikipediaSessions} Application.
+ */
+public class TopWikipediaSessionsTest
+{
+ @Test
+ public void TopWikipediaSessionsTest() throws Exception
+ {
+ LocalMode lma = LocalMode.newInstance();
+ Configuration conf = new Configuration(false);
+ conf.set("dt.application.TopWikipediaSessions.operator.console.silent", "true");
+ lma.prepareDAG(new TopWikipediaSessions(), conf);
+ LocalMode.Controller lc = lma.getController();
+
+ ((StramLocalCluster)lc).setExitCondition(new Callable<Boolean>()
+ {
+ @Override
+ public Boolean call() throws Exception
+ {
+ return TopWikipediaSessions.SessionGen.getTupleCount() >= 250;
+ }
+ });
+
+ lc.run(30000);
+
+ for (int i = 0; i < TopWikipediaSessions.Collector.getResult().size(); i++) {
+ Assert.assertTrue(isInOrder(TopWikipediaSessions.Collector.getResult().get(i)));
+ }
+ }
+
+ public boolean isInOrder(List<TopWikipediaSessions.TempWrapper> input)
+ {
+ if (input.size() == 0 || input.size() == 1) {
+ return true;
+ }
+ for (int i = 0; i < input.size() - 2; i++) {
+ if (input.get(i).getValue().getValue() < input.get(i + 1).getValue().getValue()) {
+ return false;
+ }
+ }
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/complete/TrafficRoutesTest.java
----------------------------------------------------------------------
diff --git a/examples/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/complete/TrafficRoutesTest.java b/examples/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/complete/TrafficRoutesTest.java
new file mode 100644
index 0000000..766fa60
--- /dev/null
+++ b/examples/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/complete/TrafficRoutesTest.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.malhar.stream.sample.complete;
+
+import java.util.Map;
+import java.util.concurrent.Callable;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.LocalMode;
+import com.datatorrent.lib.util.KeyValPair;
+import com.datatorrent.stram.StramLocalCluster;
+
+/**
+ * Testing the {@link TrafficRoutes} Application.
+ */
+public class TrafficRoutesTest
+{
+
+ @Test
+ public void TrafficRoutesTest() throws Exception
+ {
+ LocalMode lma = LocalMode.newInstance();
+ Configuration conf = new Configuration(false);
+ conf.set("dt.application.TrafficRoutes.operator.console.silent", "true");
+ lma.prepareDAG(new TrafficRoutes(), conf);
+ LocalMode.Controller lc = lma.getController();
+
+ ((StramLocalCluster)lc).setExitCondition(new Callable<Boolean>()
+ {
+ @Override
+ public Boolean call() throws Exception
+ {
+ return TrafficRoutes.InfoGen.getTupleCount() >= 100;
+ }
+ });
+
+ lc.run(60000);
+
+ Assert.assertTrue(!TrafficRoutes.Collector.getResult().isEmpty());
+ for (Map.Entry<KeyValPair<Long, String>, KeyValPair<Double, Boolean>> entry : TrafficRoutes.Collector.getResult().entrySet()) {
+ Assert.assertTrue(entry.getValue().getKey() <= 75);
+ Assert.assertTrue(entry.getValue().getKey() >= 55);
+ Assert.assertTrue(entry.getKey().getValue().equals("SDRoute1") || entry.getKey().getValue().equals("SDRoute2"));
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/complete/TwitterAutoCompleteTest.java
----------------------------------------------------------------------
diff --git a/examples/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/complete/TwitterAutoCompleteTest.java b/examples/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/complete/TwitterAutoCompleteTest.java
new file mode 100644
index 0000000..9ba2f25
--- /dev/null
+++ b/examples/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/complete/TwitterAutoCompleteTest.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.malhar.stream.sample.complete;
+
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.LocalMode;
+
+/**
+ * Testing the TwitterAutoComplete Application. In order to run this test, you need to create an app
+ * at https://apps.twitter.com, then generate your consumer and access keys and tokens, and set the following properties
+ * for the application before running it:
+ * Your application consumer key,
+ * Your application consumer secret,
+ * Your twitter access token, and
+ * Your twitter access token secret.
+ *
+ * This test is mainly for local demonstration purpose. Default time to run the application is 1 minute, please
+ * set the time you need to run the application before you run.
+ */
+public class TwitterAutoCompleteTest
+{
+ private static final Logger logger = LoggerFactory.getLogger(org.apache.apex.malhar.stream.sample.complete.AutoCompleteTest.class);
+
+ @Test
+ @Ignore
+ public void TwitterAutoCompleteTest() throws Exception
+ {
+ LocalMode lma = LocalMode.newInstance();
+ Configuration conf = new Configuration(false);
+ //uncomment the following lines and change YOUR_XXX to the corresponding information needed.
+ //conf.set("dt.application.TwitterAutoComplete.operator.tweetSampler.consumerKey", "YOUR_CONSUMERKEY");
+ //conf.set("dt.application.TwitterAutoComplete.operator.tweetSampler.consumerSecret", "YOUR_CONSUERSECRET");
+ //conf.set("dt.application.TwitterAutoComplete.operator.tweetSampler.accessToken", "YOUR_ACCESSTOKEN");
+ //conf.set("dt.application.TwitterAutoComplete.operator.tweetSampler.accessTokenSecret", "YOUR_TOKENSECRET");
+ lma.prepareDAG(new TwitterAutoComplete(), conf);
+ LocalMode.Controller lc = lma.getController();
+ long start = System.currentTimeMillis();
+ lc.run(60000); // Set your desired time to run the application here.
+ long end = System.currentTimeMillis();
+ long time = end - start;
+ logger.info("Test used " + time + " ms");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/cookbook/CombinePerKeyExamplesTest.java
----------------------------------------------------------------------
diff --git a/examples/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/cookbook/CombinePerKeyExamplesTest.java b/examples/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/cookbook/CombinePerKeyExamplesTest.java
new file mode 100644
index 0000000..1e14fff
--- /dev/null
+++ b/examples/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/cookbook/CombinePerKeyExamplesTest.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.malhar.stream.sample.cookbook;
+
+import java.util.concurrent.Callable;
+import org.junit.Assert;
+import org.junit.Test;
+import org.apache.hadoop.conf.Configuration;
+import com.datatorrent.api.LocalMode;
+import com.datatorrent.stram.StramLocalCluster;
+
+/**
+ * Test for {@link CombinePerKeyExamples}.
+ */
+public class CombinePerKeyExamplesTest
+{
+ @Test
+ public void CombinePerKeyExamplesTest() throws Exception
+ {
+ LocalMode lma = LocalMode.newInstance();
+ Configuration conf = new Configuration(false);
+ conf.set("dt.application.CombinePerKeyExamples.operator.console.silent", "true");
+ CombinePerKeyExamples app = new CombinePerKeyExamples();
+
+ lma.prepareDAG(app, conf);
+
+ LocalMode.Controller lc = lma.getController();
+ ((StramLocalCluster)lc).setExitCondition(new Callable<Boolean>()
+ {
+ @Override
+ public Boolean call() throws Exception
+ {
+ return CombinePerKeyExamples.Collector.isDone();
+ }
+ });
+ lc.run(100000);
+
+ Assert.assertTrue(CombinePerKeyExamples.Collector.getResult().get(CombinePerKeyExamples.Collector.getResult().size() - 2).getCorpus().contains("1, 2, 3, 4, 5, 6, 7, 8"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/cookbook/DeDupExampleTest.java
----------------------------------------------------------------------
diff --git a/examples/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/cookbook/DeDupExampleTest.java b/examples/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/cookbook/DeDupExampleTest.java
new file mode 100644
index 0000000..7f93f50
--- /dev/null
+++ b/examples/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/cookbook/DeDupExampleTest.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.malhar.stream.sample.cookbook;
+
+import java.util.concurrent.Callable;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.apache.hadoop.conf.Configuration;
+import com.datatorrent.api.LocalMode;
+
+import com.datatorrent.stram.StramLocalCluster;
+
+
+/**
+ * Test for {@link DeDupExample}.
+ */
+public class DeDupExampleTest
+{
+ @Test
+ public void DeDupExampleTest() throws Exception
+ {
+ LocalMode lma = LocalMode.newInstance();
+ Configuration conf = new Configuration(false);
+ conf.set("dt.application.DeDupExample.operator.console.silent", "true");
+ DeDupExample app = new DeDupExample();
+ lma.prepareDAG(app, conf);
+ LocalMode.Controller lc = lma.getController();
+ ((StramLocalCluster)lc).setExitCondition(new Callable<Boolean>()
+ {
+ @Override
+ public Boolean call() throws Exception
+ {
+ return DeDupExample.Collector.isDone();
+ }
+ });
+ lc.run(50000);
+
+ Assert.assertEquals(9, DeDupExample.Collector.getResult().getValue().size());
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/cookbook/MaxPerKeyExamplesTest.java
----------------------------------------------------------------------
diff --git a/examples/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/cookbook/MaxPerKeyExamplesTest.java b/examples/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/cookbook/MaxPerKeyExamplesTest.java
new file mode 100644
index 0000000..ec28b40
--- /dev/null
+++ b/examples/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/cookbook/MaxPerKeyExamplesTest.java
@@ -0,0 +1,210 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.malhar.stream.sample.cookbook;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.Callable;
+
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.google.common.base.Throwables;
+
+import com.datatorrent.api.LocalMode;
+import com.datatorrent.lib.db.jdbc.JdbcTransactionalStore;
+import com.datatorrent.stram.StramLocalCluster;
+
+/**
+ * Test for MaxPerKeyExamples Application.
+ */
+public class MaxPerKeyExamplesTest
+{
+
+ private static final String INPUT_TUPLE_CLASS = "org.apache.apex.malhar.stream.sample.cookbook.InputPojo";
+ private static final String OUTPUT_TUPLE_CLASS = "org.apache.apex.malhar.stream.sample.cookbook.OutputPojo";
+ private static final String DB_DRIVER = "org.h2.Driver";
+ private static final String DB_URL = "jdbc:h2:~/test";
+ private static final String INPUT_TABLE = "InputTable";
+ private static final String OUTPUT_TABLE = "OutputTable";
+ private static final String USER_NAME = "root";
+ private static final String PSW = "password";
+ private static final String QUERY = "SELECT * FROM " + INPUT_TABLE + ";";
+
+ private static final double[] MEANTEMPS = {85.3, 75.4};
+
+ @BeforeClass
+ public static void setup()
+ {
+ try {
+ Class.forName(DB_DRIVER).newInstance();
+
+ Connection con = DriverManager.getConnection(DB_URL,USER_NAME,PSW);
+ Statement stmt = con.createStatement();
+
+ String createMetaTable = "CREATE TABLE IF NOT EXISTS " + JdbcTransactionalStore.DEFAULT_META_TABLE + " ( "
+ + JdbcTransactionalStore.DEFAULT_APP_ID_COL + " VARCHAR(100) NOT NULL, "
+ + JdbcTransactionalStore.DEFAULT_OPERATOR_ID_COL + " INT NOT NULL, "
+ + JdbcTransactionalStore.DEFAULT_WINDOW_COL + " BIGINT NOT NULL, "
+ + "UNIQUE (" + JdbcTransactionalStore.DEFAULT_APP_ID_COL + ", "
+ + JdbcTransactionalStore.DEFAULT_OPERATOR_ID_COL + ", " + JdbcTransactionalStore.DEFAULT_WINDOW_COL + ") "
+ + ")";
+ stmt.executeUpdate(createMetaTable);
+
+ String createInputTable = "CREATE TABLE IF NOT EXISTS " + INPUT_TABLE
+ + "(MONTH INT(2) not NULL, DAY INT(2), YEAR INT(4), MEANTEMP DOUBLE(10) )";
+ stmt.executeUpdate(createInputTable);
+
+ String createOutputTable = "CREATE TABLE IF NOT EXISTS " + OUTPUT_TABLE
+ + "(MONTH INT(2) not NULL, MEANTEMP DOUBLE(10) )";
+ stmt.executeUpdate(createOutputTable);
+
+ String cleanTable = "truncate table " + INPUT_TABLE;
+ stmt.executeUpdate(cleanTable);
+
+ stmt = con.createStatement();
+
+ String sql = "INSERT INTO " + INPUT_TABLE + " VALUES (6, 21, 2014, 85.3)";
+ stmt.executeUpdate(sql);
+ sql = "INSERT INTO " + INPUT_TABLE + " VALUES (7, 20, 2014, 75.4)";
+ stmt.executeUpdate(sql);
+ sql = "INSERT INTO " + INPUT_TABLE + " VALUES (6, 18, 2014, 45.3)";
+ stmt.executeUpdate(sql);
+
+ } catch (Throwable e) {
+ throw Throwables.propagate(e);
+ }
+ }
+
+ @AfterClass
+ public static void cleanup()
+ {
+ try {
+ Class.forName(DB_DRIVER).newInstance();
+
+ Connection con = DriverManager.getConnection(DB_URL, USER_NAME, PSW);
+ Statement stmt = con.createStatement();
+
+ String dropInputTable = "DROP TABLE " + INPUT_TABLE;
+ stmt.executeUpdate(dropInputTable);
+
+ String dropOutputTable = "DROP TABLE " + OUTPUT_TABLE;
+ stmt.executeUpdate(dropOutputTable);
+
+ } catch (Throwable e) {
+ throw Throwables.propagate(e);
+ }
+
+ }
+
+ public void setConfig(Configuration conf)
+ {
+ conf.set("dt.operator.jdbcInput.prop.store.userName", USER_NAME);
+ conf.set("dt.operator.jdbcInput.prop.store.password", PSW);
+ conf.set("dt.operator.jdbcInput.prop.store.databaseDriver", DB_DRIVER);
+ conf.set("dt.operator.jdbcInput.prop.batchSize", "5");
+ conf.set("dt.operator.jdbcInput.port.outputPort.attr.TUPLE_CLASS", INPUT_TUPLE_CLASS);
+ conf.set("dt.operator.jdbcInput.prop.store.databaseUrl", DB_URL);
+ conf.set("dt.operator.jdbcInput.prop.tableName", INPUT_TABLE);
+ conf.set("dt.operator.jdbcInput.prop.query", QUERY);
+
+ conf.set("dt.operator.jdbcOutput.prop.store.userName", USER_NAME);
+ conf.set("dt.operator.jdbcOutput.prop.store.password", PSW);
+ conf.set("dt.operator.jdbcOutput.prop.store.databaseDriver", DB_DRIVER);
+ conf.set("dt.operator.jdbcOutput.prop.batchSize", "5");
+ conf.set("dt.operator.jdbcOutput.port.input.attr.TUPLE_CLASS", OUTPUT_TUPLE_CLASS);
+ conf.set("dt.operator.jdbcOutput.prop.store.databaseUrl", DB_URL);
+ conf.set("dt.operator.jdbcOutput.prop.tablename", OUTPUT_TABLE);
+ }
+
+ public int getNumEntries()
+ {
+ Connection con;
+ try {
+ con = DriverManager.getConnection(DB_URL,USER_NAME,PSW);
+ Statement stmt = con.createStatement();
+
+ String countQuery = "SELECT count(DISTINCT (MONTH, MEANTEMP)) from " + OUTPUT_TABLE;
+ ResultSet resultSet = stmt.executeQuery(countQuery);
+ resultSet.next();
+ return resultSet.getInt(1);
+ } catch (SQLException e) {
+ throw new RuntimeException("fetching count", e);
+ }
+ }
+
+ public Map<Integer, Double> getMaxMeanTemp()
+ {
+ Map<Integer, Double> result = new HashMap<>();
+ Connection con;
+ try {
+ con = DriverManager.getConnection(DB_URL,USER_NAME,PSW);
+ Statement stmt = con.createStatement();
+
+ String countQuery = "SELECT DISTINCT * from " + OUTPUT_TABLE;
+ ResultSet resultSet = stmt.executeQuery(countQuery);
+ while (resultSet.next()) {
+ result.put(resultSet.getInt("MONTH"), resultSet.getDouble("MEANTEMP"));
+
+ }
+ return result;
+ } catch (SQLException e) {
+ throw new RuntimeException("fetching count", e);
+ }
+ }
+
+ @Test
+ public void MaxPerKeyExampleTest() throws Exception
+ {
+ LocalMode lma = LocalMode.newInstance();
+ Configuration conf = new Configuration(false);
+ setConfig(conf);
+
+ MaxPerKeyExamples app = new MaxPerKeyExamples();
+
+ lma.prepareDAG(app, conf);
+
+ LocalMode.Controller lc = lma.getController();
+ ((StramLocalCluster)lc).setExitCondition(new Callable<Boolean>()
+ {
+ @Override
+ public Boolean call() throws Exception
+ {
+ return getNumEntries() == 2;
+ }
+ });
+
+ lc.run(5000);
+
+ double[] result = new double[2];
+ result[0] = getMaxMeanTemp().get(6);
+ result[1] = getMaxMeanTemp().get(7);
+ Assert.assertArrayEquals(MEANTEMPS, result, 0.0);
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/highlevelapi/src/test/resources/data/word.txt
----------------------------------------------------------------------
diff --git a/examples/highlevelapi/src/test/resources/data/word.txt b/examples/highlevelapi/src/test/resources/data/word.txt
new file mode 100644
index 0000000..7e28409
--- /dev/null
+++ b/examples/highlevelapi/src/test/resources/data/word.txt
@@ -0,0 +1,2 @@
+word1 word2 word7 word9 word2 word3 word2 word4 word7 word3 word9 word9 word5 word5 word4 word2 word1 error
+word1 word2 word7 word9 word2 word3 word2 word4 word7 word3 word9 word9 word5 word5 word4 word2 word1 error
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/highlevelapi/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/examples/highlevelapi/src/test/resources/log4j.properties b/examples/highlevelapi/src/test/resources/log4j.properties
new file mode 100644
index 0000000..592eb19
--- /dev/null
+++ b/examples/highlevelapi/src/test/resources/log4j.properties
@@ -0,0 +1,45 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+log4j.rootLogger=INFO,CONSOLE
+
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+log4j.appender.CONSOLE.threshold=INFO
+#log4j.appender.CONSOLE.threshold=${test.log.console.threshold}
+test.log.console.threshold=WARN
+
+log4j.appender.RFA=org.apache.log4j.RollingFileAppender
+log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
+log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+log4j.appender.RFA.File=/tmp/app.log
+
+# to enable, add SYSLOG to rootLogger
+log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender
+log4j.appender.SYSLOG.syslogHost=127.0.0.1
+log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout
+log4j.appender.SYSLOG.layout.conversionPattern=${dt.cid} %-5p [%t] %c{2} %x - %m%n
+log4j.appender.SYSLOG.Facility=LOCAL1
+
+#log4j.logger.org=INFO
+
+#log4j.logger.org.apache.commons.beanutils=warn
+log4j.logger.com.datatorrent=INFO
+log4j.logger.org.apache.apex=INFO
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/highlevelapi/src/test/resources/sampletweets.txt
----------------------------------------------------------------------
diff --git a/examples/highlevelapi/src/test/resources/sampletweets.txt b/examples/highlevelapi/src/test/resources/sampletweets.txt
new file mode 100644
index 0000000..c113130
--- /dev/null
+++ b/examples/highlevelapi/src/test/resources/sampletweets.txt
@@ -0,0 +1,44 @@
+Last week, I published a blog announcing that Apex was accepted as an Apache incubator project.
+This week, I\u2019ll give you a little more detail on what Apache Apex is, and why it\u2019s important.
+
+Apache #Hadoop has been around for over a decade. It has become the de-facto big data platform,
+allowing enterprises to transform their business operations by turning big data into something useful, meaningful,
+and revenue-generating. #Hadoop promised the enablement of big data without incurring the costs you would normally think such powerful
+processing systems would demand. This tremendous promise of transforming business operations continues to fuel high growth in the industry.
+
+It all got started when Hadoop engineers at Yahoo! asked, \u201cHow can we build an efficient search indexing capability?\u201d
+The ensuing iterations and some inspiration resulted in the #MapReduce programming model. Although powerful, MapReduce wasn\u2019t perfect.
+
+Mastering MapReduce required a steep learning curve. Migrating applications to MapReduce required an almost complete rewrite.
+Equally worrisome was the fact that MapReduce had batch processing paradigm and \u201ccompute going to data\u201d at its core,
+thus posing a deterrent to Hadoop realizing its true potential.
+
+Expectedly enough, #MapReduce was an impediment that did little to bolster productization of big data.
+Not to be deterred, there were faster substitutes for MapReduce. Just like Hadoop, these models required deeper expertise, were tough to operate and difficult to master.
+As such, #Hadoop disrupted the way big data needs were handled, but remained largely under-productized.
+
+A decade after Hadoop was started, only a small percentage of big data projects are in production.
+Data is growing rapidly and the ability to harness big data has become a decisive competitive advantage.
+MapReduce impedes this demand (actually more of a scramble) to transform into a data-driven business.
+
+In hindsight, it is clear that in the early days, the subsequent success of Hadoop was not anticipated.
+If they had anticipated Hadoop\u2019s success, the question would have been, \u201cWhat can we do with massively distributed resources?\u201d
+The answer to this question, which came about soon after, was YARN (Hadoop 2.0), the next generation Hadoop.
+For the first time, #YARN brought the capability of exploring how distributed resources handling big data could perform \u201ca lot of things\u201d,
+thus going beyond the early MapReduce paradigm, and in a way beyond batch or even compute-going-to-data paradigms.
+YARN presented the capability to allow big data to not just become big in size, but broader in use cases. With its enabling capability as a Hadoop facilitator,
+YARN has pushed Hadoop towards realizing its true potential. The Hadoop predicament is similar to what
+cellphones would have been without the more popular features such as messaging and internet connectivity.
+
+
+In their early years, cellphones upset the landline market, but did not foster an immediate market furor till
+it transformed into the new-age \u201csmartphone\u201d with impressive features.
+YARN is most certainly the enabling factor for big data dreaming bigger and wider, and with it, Hadoop 2.0 is now a true de-facto distributed operating system.
+
+What\u2019s needed is bleeding edge YARN-based platforms capable of radically realizing Hadoop\u2019s potential
+
+Now is the right time to not only productize big data, but to see how setting it in motion can ensure realization of greater business goals.
+A Herculean task, this demands platforms that are easy to deploy, require nothing beyond everyday IT expertise, can effortlessly integrate with an existing IT infrastructure while ensuring ease of migration.
+The new-age Hadoop platforms need to be designed with an approach to reduce time-to-market by shortening the application lifecycle, from building to launching, thus quickening the realization of revenue for businesses.
+They will also have to reduce time for developers to develop, devOps to operationalize, and finally reduce time to insight for business.
+Platforms such as these will need to learn, adapt, and change to meet the burgeoning needs of the big data world.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/highlevelapi/src/test/resources/wordcount/word.txt
----------------------------------------------------------------------
diff --git a/examples/highlevelapi/src/test/resources/wordcount/word.txt b/examples/highlevelapi/src/test/resources/wordcount/word.txt
new file mode 100644
index 0000000..edd0f51
--- /dev/null
+++ b/examples/highlevelapi/src/test/resources/wordcount/word.txt
@@ -0,0 +1,8 @@
+word1 word2 word7 word9 word2 word3 word2 word4 word7 word3 word9 word9 word5 word5 word4 word2 word1 error
+word1 word2 word7 word9 word2 word3 word2 word4 word7 word3 word9 word9 word5 word5 word4 word2 word1 error
+word1 word2 word7 word9 word2 word3 word2 word4 word7 word3 word9 word9 word5 word5 word4 word2 word1 error
+word1 word2 word7 word9 word2 word3 word2 word4 word7 word3 word9 word9 word5 word5 word4 word2 word1 error
+word1 word2 word7 word9 word2 word3 word2 word4 word7 word3 word9 word9 word5 word5 word4 word2 word1 error
+word1 word2 word7 word9 word2 word3 word2 word4 word7 word3 word9 word9 word5 word5 word4 word2 word1 error
+word1 word2 word7 word9 word2 word3 word2 word4 word7 word3 word9 word9 word5 word5 word4 word2 word1 error
+bye
\ No newline at end of file
[07/30] apex-malhar git commit: Renamed demos to examples. Packages
and artifactid names are changed as suggested.
Posted by th...@apache.org.
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/mobile/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/examples/mobile/src/assemble/appPackage.xml b/examples/mobile/src/assemble/appPackage.xml
new file mode 100644
index 0000000..4138cf2
--- /dev/null
+++ b/examples/mobile/src/assemble/appPackage.xml
@@ -0,0 +1,59 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+ <id>appPackage</id>
+ <formats>
+ <format>jar</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>${basedir}/target/</directory>
+ <outputDirectory>/app</outputDirectory>
+ <includes>
+ <include>${project.artifactId}-${project.version}.jar</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/target/deps</directory>
+ <outputDirectory>/lib</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/site/conf</directory>
+ <outputDirectory>/conf</outputDirectory>
+ <includes>
+ <include>*.xml</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/META-INF</directory>
+ <outputDirectory>/META-INF</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/app</directory>
+ <outputDirectory>/app</outputDirectory>
+ </fileSet>
+ </fileSets>
+
+</assembly>
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/mobile/src/main/java/org/apache/apex/examples/mobile/Application.java
----------------------------------------------------------------------
diff --git a/examples/mobile/src/main/java/org/apache/apex/examples/mobile/Application.java b/examples/mobile/src/main/java/org/apache/apex/examples/mobile/Application.java
new file mode 100644
index 0000000..f719643
--- /dev/null
+++ b/examples/mobile/src/main/java/org/apache/apex/examples/mobile/Application.java
@@ -0,0 +1,172 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.mobile;
+
+import java.net.URI;
+import java.util.Arrays;
+import java.util.Map;
+import java.util.Random;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.lang.mutable.MutableLong;
+import org.apache.commons.lang3.Range;
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.Context.OperatorContext;
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.StatsListener;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.lib.counters.BasicCounters;
+import com.datatorrent.lib.io.PubSubWebSocketInputOperator;
+import com.datatorrent.lib.io.PubSubWebSocketOutputOperator;
+import com.datatorrent.lib.partitioner.StatelessThroughputBasedPartitioner;
+import com.datatorrent.lib.testbench.RandomEventGenerator;
+
+/**
+ * Mobile Example Application:
+ * <p>
+ * This example simulates large number of cell phones in the range of 40K to 200K
+ * and tracks a given cell number across cell towers. It also displays the changing locations of the cell number on a google map.
+ *
+ * This example demonstrates the scalability feature of the Apex platform.
+ * It showcases the ability of the platform to scale up and down as the phone numbers generated increase and decrease respectively.
+ * If the tuples processed per second by the pmove operator increase beyond 30,000, more partitions of the pmove operator gets deployed until
+ * each of the partition processes around 10000 to 30000 tuples per second.
+ * If the tuples processed per second drops below 10,000, the platform merges the operators until the partition count drops down to the original.
+ * The load can be varied using the tuplesBlast property.
+ * If the tuplesBlast is set to 200, 40K cell phones are generated.
+ * If the tuplesBlast is set to 1000, 200K cell phones are generated.
+ * The tuplesBlast property can be set using dtcli command: 'set-operator-property pmove tuplesBlast 1000'.
+ *
+ *
+ * The specs are as such<br>
+ * Depending on the tuplesBlast property, large number of cell phone numbers are generated.
+ * They jump a cell tower frequently. Sometimes
+ * within a second sometimes in 10 seconds. The aim is to demonstrate the
+ * following abilities<br>
+ * <ul>
+ * <li>Entering query dynamically: The phone numbers are added to locate its gps
+ * in run time.</li>
+ * <li>Changing functionality dynamically: The load is changed by making
+ * functional changes on the load generator operator (phonegen)(</li>
+ * <li>Auto Scale up/Down with load: Operator pmove increases and decreases
+ * partitions as per load</li>
+ * <li></li>
+ * </ul>
+ *
+ * Refer to examples/docs/MobileDemo.md for more information.
+ *
+ * <p>
+ *
+ * Running Java Test or Main app in IDE:
+ *
+ * <pre>
+ * LocalMode.runApp(new Application(), 600000); // 10 min run
+ * </pre>
+ *
+ * Run Success : <br>
+ * For successful deployment and run, user should see following output on
+ * console: <br>
+ *
+ * <pre>
+ * phoneLocationQueryResult: {phone=5556101, location=(5,9), queryId=q3}
+ * phoneLocationQueryResult: {phone=5554995, location=(10,4), queryId=q1}
+ * phoneLocationQueryResult: {phone=5556101, location=(5,9), queryId=q3}
+ * phoneLocationQueryResult: {phone=5554995, location=(10,4), queryId=q1}
+ * phoneLocationQueryResult: {phone=5554995, location=(10,5), queryId=q1}
+ * phoneLocationQueryResult: {phone=5556101, location=(5,9), queryId=q3}
+ * phoneLocationQueryResult: {phone=5554995, location=(9,5), queryId=q1}
+ * phoneLocationQueryResult: {phone=5556101, location=(5,9), queryId=q3}
+ * phoneLocationQueryResult: {phone=5556101, location=(5,9), queryId=q3}
+ * phoneLocationQueryResult: {phone=5554995, location=(9,5), queryId=q1}
+ * phoneLocationQueryResult: {phone=5554995, location=(9,5), queryId=q1}
+ * phoneLocationQueryResult: {phone=5556101, location=(5,9), queryId=q3}
+ * </pre>
+ *
+ * * <b>Application DAG : </b><br>
+ * <img src="doc-files/mobile.png" width=600px > <br>
+ *
+ * @since 0.3.2
+ */
+@ApplicationAnnotation(name = "MobileExample")
+public class Application implements StreamingApplication
+{
+ public static final String PHONE_RANGE_PROP = "dt.application.MobileExample.phoneRange";
+ public static final String TOTAL_SEED_NOS = "dt.application.MobileExample.totalSeedNumbers";
+ public static final String COOL_DOWN_MILLIS = "dt.application.MobileExample.coolDownMillis";
+ public static final String MAX_THROUGHPUT = "dt.application.MobileExample.maxThroughput";
+ public static final String MIN_THROUGHPUT = "dt.application.MobileExample.minThroughput";
+ private static final Logger LOG = LoggerFactory.getLogger(Application.class);
+ private Range<Integer> phoneRange = Range.between(5550000, 5559999);
+
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ String lPhoneRange = conf.get(PHONE_RANGE_PROP, null);
+ if (lPhoneRange != null) {
+ String[] tokens = lPhoneRange.split("-");
+ if (tokens.length != 2) {
+ throw new IllegalArgumentException("Invalid range: " + lPhoneRange);
+ }
+ this.phoneRange = Range.between(Integer.parseInt(tokens[0]), Integer.parseInt(tokens[1]));
+ }
+ LOG.debug("Phone range {}", this.phoneRange);
+
+ RandomEventGenerator phones = dag.addOperator("Receiver", RandomEventGenerator.class);
+ phones.setMinvalue(this.phoneRange.getMinimum());
+ phones.setMaxvalue(this.phoneRange.getMaximum());
+
+ PhoneMovementGenerator movementGen = dag.addOperator("LocationFinder", PhoneMovementGenerator.class);
+ dag.setAttribute(movementGen, OperatorContext.COUNTERS_AGGREGATOR, new BasicCounters.LongAggregator<MutableLong>());
+
+ StatelessThroughputBasedPartitioner<PhoneMovementGenerator> partitioner = new StatelessThroughputBasedPartitioner<PhoneMovementGenerator>();
+ partitioner.setCooldownMillis(conf.getLong(COOL_DOWN_MILLIS, 45000));
+ partitioner.setMaximumEvents(conf.getLong(MAX_THROUGHPUT, 30000));
+ partitioner.setMinimumEvents(conf.getLong(MIN_THROUGHPUT, 10000));
+ dag.setAttribute(movementGen, OperatorContext.STATS_LISTENERS, Arrays.asList(new StatsListener[]{partitioner}));
+ dag.setAttribute(movementGen, OperatorContext.PARTITIONER, partitioner);
+
+ // generate seed numbers
+ Random random = new Random();
+ int maxPhone = phoneRange.getMaximum() - phoneRange.getMinimum();
+ int phonesToDisplay = conf.getInt(TOTAL_SEED_NOS, 10);
+ for (int i = phonesToDisplay; i-- > 0; ) {
+ int phoneNo = phoneRange.getMinimum() + random.nextInt(maxPhone + 1);
+ LOG.info("seed no: " + phoneNo);
+ movementGen.phoneRegister.add(phoneNo);
+ }
+ // done generating data
+ LOG.info("Finished generating seed data.");
+
+ String gatewayAddress = dag.getValue(DAG.GATEWAY_CONNECT_ADDRESS);
+ URI uri = URI.create("ws://" + gatewayAddress + "/pubsub");
+ PubSubWebSocketOutputOperator<Object> wsOut = dag.addOperator("LocationResults", new PubSubWebSocketOutputOperator<Object>());
+ wsOut.setUri(uri);
+ PubSubWebSocketInputOperator<Map<String, String>> wsIn = dag.addOperator("QueryLocation", new PubSubWebSocketInputOperator<Map<String, String>>());
+ wsIn.setUri(uri);
+ // default partitioning: first connected stream to movementGen will be partitioned
+ dag.addStream("Phone-Data", phones.integer_data, movementGen.data);
+ dag.addStream("Results", movementGen.locationQueryResult, wsOut.input);
+ dag.addStream("Query", wsIn.outputPort, movementGen.phoneQuery);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/mobile/src/main/java/org/apache/apex/examples/mobile/PhoneEntryOperator.java
----------------------------------------------------------------------
diff --git a/examples/mobile/src/main/java/org/apache/apex/examples/mobile/PhoneEntryOperator.java b/examples/mobile/src/main/java/org/apache/apex/examples/mobile/PhoneEntryOperator.java
new file mode 100644
index 0000000..f8de357
--- /dev/null
+++ b/examples/mobile/src/main/java/org/apache/apex/examples/mobile/PhoneEntryOperator.java
@@ -0,0 +1,124 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.mobile;
+
+import java.util.Map;
+import java.util.Random;
+import javax.validation.constraints.Min;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Maps;
+import com.google.common.collect.Range;
+
+import com.datatorrent.api.DefaultInputPort;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.annotation.InputPortFieldAnnotation;
+import com.datatorrent.common.util.BaseOperator;
+
+/**
+ * Generates mobile numbers that will be displayed in mobile example just after launch.<br></br>
+ * Operator attributes:<b>
+ * <ul>
+ * <li>initialDisplayCount: No. of seed phone numbers that will be generated.</li>
+ * <li>maxSeedPhoneNumber: The largest seed phone number.</li>
+ * </ul>
+ * </b>
+ *
+ * @since 0.3.5
+ */
+public class PhoneEntryOperator extends BaseOperator
+{
+ private static Logger LOG = LoggerFactory.getLogger(PhoneEntryOperator.class);
+
+ private boolean seedGenerationDone = false;
+
+ @Min(0)
+ private int initialDisplayCount = 0;
+
+ private int maxSeedPhoneNumber = 0;
+ private int rangeLowerEndpoint;
+ private int rangeUpperEndpoint;
+
+ /**
+ * Sets the initial number of phones to display on the google map.
+ *
+ * @param i the count of initial phone numbers to display
+ */
+ public void setInitialDisplayCount(int i)
+ {
+ initialDisplayCount = i;
+ }
+
+ /**
+ * Sets the range for the phone numbers generated by the operator.
+ *
+ * @param i the range within which the phone numbers are randomly generated.
+ */
+ public void setPhoneRange(Range<Integer> phoneRange)
+ {
+ this.rangeLowerEndpoint = phoneRange.lowerEndpoint();
+ this.rangeUpperEndpoint = phoneRange.upperEndpoint();
+ }
+
+ /**
+ * Sets the max seed for random phone number generation
+ *
+ * @param i the number to initialize the random number phone generator.
+ */
+ public void setMaxSeedPhoneNumber(int number)
+ {
+ this.maxSeedPhoneNumber = number;
+ }
+
+ @InputPortFieldAnnotation(optional = true)
+ public final transient DefaultInputPort<Map<String, String>> locationQuery = new DefaultInputPort<Map<String, String>>()
+ {
+ @Override
+ public void process(Map<String, String> tuple)
+ {
+ seedPhones.emit(tuple);
+ }
+ };
+
+ public final transient DefaultOutputPort<Map<String, String>> seedPhones = new DefaultOutputPort<Map<String, String>>();
+
+ @Override
+ public void beginWindow(long windowId)
+ {
+ if (!seedGenerationDone) {
+ Random random = new Random();
+ int maxPhone = (maxSeedPhoneNumber <= rangeUpperEndpoint && maxSeedPhoneNumber >= rangeLowerEndpoint) ? maxSeedPhoneNumber : rangeUpperEndpoint;
+ maxPhone -= 5550000;
+ int phonesToDisplay = initialDisplayCount > maxPhone ? maxPhone : initialDisplayCount;
+ for (int i = phonesToDisplay; i-- > 0; ) {
+ int phoneNo = 5550000 + random.nextInt(maxPhone + 1);
+ LOG.info("seed no: " + phoneNo);
+ Map<String, String> valueMap = Maps.newHashMap();
+ valueMap.put(PhoneMovementGenerator.KEY_COMMAND, PhoneMovementGenerator.COMMAND_ADD);
+ valueMap.put(PhoneMovementGenerator.KEY_PHONE, Integer.toString(phoneNo));
+ seedPhones.emit(valueMap);
+ }
+ // done generating data
+ seedGenerationDone = true;
+ LOG.info("Finished generating seed data.");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/mobile/src/main/java/org/apache/apex/examples/mobile/PhoneMovementGenerator.java
----------------------------------------------------------------------
diff --git a/examples/mobile/src/main/java/org/apache/apex/examples/mobile/PhoneMovementGenerator.java b/examples/mobile/src/main/java/org/apache/apex/examples/mobile/PhoneMovementGenerator.java
new file mode 100644
index 0000000..3a293f0
--- /dev/null
+++ b/examples/mobile/src/main/java/org/apache/apex/examples/mobile/PhoneMovementGenerator.java
@@ -0,0 +1,335 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.mobile;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+
+import javax.validation.constraints.Min;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.lang.mutable.MutableLong;
+
+import com.google.common.base.Strings;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+
+import com.datatorrent.api.Context.OperatorContext;
+import com.datatorrent.api.DefaultInputPort;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.annotation.InputPortFieldAnnotation;
+import com.datatorrent.common.util.BaseOperator;
+import com.datatorrent.lib.counters.BasicCounters;
+import com.datatorrent.lib.util.HighLow;
+
+/**
+ * <p>
+ * This operator generates the GPS locations for the phone numbers specified.
+ * The range of phone numbers or a specific phone number can be set for which the GPS locations will be generated.
+ * It supports querying the locations of a given phone number.
+ * This is a partionable operator that can partition as the tuplesBlast increases.
+ * </p>
+ *
+ * @since 0.3.2
+ */
+public class PhoneMovementGenerator extends BaseOperator
+{
+ public final transient DefaultInputPort<Integer> data = new DefaultInputPort<Integer>()
+ {
+ @Override
+ public void process(Integer tuple)
+ {
+ HighLow<Integer> loc = gps.get(tuple);
+ if (loc == null) {
+ loc = new HighLow<Integer>(random.nextInt(range), random.nextInt(range));
+ gps.put(tuple, loc);
+ }
+ int xloc = loc.getHigh();
+ int yloc = loc.getLow();
+ int state = rotate % 4;
+
+ // Compute new location
+ int delta = random.nextInt(100);
+ if (delta >= threshold) {
+ if (state < 2) {
+ xloc++;
+ } else {
+ xloc--;
+ }
+ if (xloc < 0) {
+ xloc += range;
+ }
+ }
+ delta = random.nextInt(100);
+ if (delta >= threshold) {
+ if ((state == 1) || (state == 3)) {
+ yloc++;
+ } else {
+ yloc--;
+ }
+ if (yloc < 0) {
+ yloc += range;
+ }
+ }
+ xloc %= range;
+ yloc %= range;
+
+ // Set new location
+ HighLow<Integer> nloc = newgps.get(tuple);
+ if (nloc == null) {
+ newgps.put(tuple, new HighLow<Integer>(xloc, yloc));
+ } else {
+ nloc.setHigh(xloc);
+ nloc.setLow(yloc);
+ }
+ rotate++;
+ }
+ };
+
+ @InputPortFieldAnnotation(optional = true)
+ public final transient DefaultInputPort<Map<String,String>> phoneQuery = new DefaultInputPort<Map<String,String>>()
+ {
+ @Override
+ public void process(Map<String,String> tuple)
+ {
+ LOG.info("new query {}", tuple);
+ String command = tuple.get(KEY_COMMAND);
+ if (command != null) {
+ if (command.equals(COMMAND_ADD)) {
+ commandCounters.getCounter(CommandCounters.ADD).increment();
+ String phoneStr = tuple.get(KEY_PHONE);
+ registerPhone(phoneStr);
+ } else if (command.equals(COMMAND_ADD_RANGE)) {
+ commandCounters.getCounter(CommandCounters.ADD_RANGE).increment();
+ registerPhoneRange(tuple.get(KEY_START_PHONE), tuple.get(KEY_END_PHONE));
+ } else if (command.equals(COMMAND_DELETE)) {
+ commandCounters.getCounter(CommandCounters.DELETE).increment();
+ String phoneStr = tuple.get(KEY_PHONE);
+ deregisterPhone(phoneStr);
+ } else if (command.equals(COMMAND_CLEAR)) {
+ commandCounters.getCounter(CommandCounters.CLEAR).increment();
+ clearPhones();
+ }
+ }
+ }
+ };
+
+ public static final String KEY_COMMAND = "command";
+ public static final String KEY_PHONE = "phone";
+ public static final String KEY_LOCATION = "location";
+ public static final String KEY_REMOVED = "removed";
+ public static final String KEY_START_PHONE = "startPhone";
+ public static final String KEY_END_PHONE = "endPhone";
+
+ public static final String COMMAND_ADD = "add";
+ public static final String COMMAND_ADD_RANGE = "addRange";
+ public static final String COMMAND_DELETE = "del";
+ public static final String COMMAND_CLEAR = "clear";
+
+ final Set<Integer> phoneRegister = Sets.newHashSet();
+
+ private final transient HashMap<Integer, HighLow<Integer>> gps = new HashMap<Integer, HighLow<Integer>>();
+ private final Random random = new Random();
+ private int range = 50;
+ private int threshold = 80;
+ private int rotate = 0;
+
+ protected BasicCounters<MutableLong> commandCounters;
+
+ private transient OperatorContext context;
+ private final transient HashMap<Integer, HighLow<Integer>> newgps = new HashMap<Integer, HighLow<Integer>>();
+
+ public PhoneMovementGenerator()
+ {
+ this.commandCounters = new BasicCounters<MutableLong>(MutableLong.class);
+ }
+
+ /**
+ * @return the range of the phone numbers
+ */
+ @Min(0)
+ public int getRange()
+ {
+ return range;
+ }
+
+ /**
+ * Sets the range of phone numbers for which the GPS locations need to be generated.
+ *
+ * @param i the range of phone numbers to set
+ */
+ public void setRange(int i)
+ {
+ range = i;
+ }
+
+ /**
+ * @return the threshold
+ */
+ @Min(0)
+ public int getThreshold()
+ {
+ return threshold;
+ }
+
+ /**
+ * Sets the threshold that decides how frequently the GPS locations are updated.
+ *
+ * @param i the value that decides how frequently the GPS locations change.
+ */
+ public void setThreshold(int i)
+ {
+ threshold = i;
+ }
+
+ private void registerPhone(String phoneStr)
+ {
+ // register the phone channel
+ if (Strings.isNullOrEmpty(phoneStr)) {
+ return;
+ }
+ try {
+ Integer phone = new Integer(phoneStr);
+ registerSinglePhone(phone);
+ } catch (NumberFormatException nfe) {
+ LOG.warn("Invalid no {}", phoneStr);
+ }
+ }
+
+ private void registerPhoneRange(String startPhoneStr, String endPhoneStr)
+ {
+ if (Strings.isNullOrEmpty(startPhoneStr) || Strings.isNullOrEmpty(endPhoneStr)) {
+ LOG.warn("Invalid phone range {} {}", startPhoneStr, endPhoneStr);
+ return;
+ }
+ try {
+ Integer startPhone = new Integer(startPhoneStr);
+ Integer endPhone = new Integer(endPhoneStr);
+ if (endPhone < startPhone) {
+ LOG.warn("Invalid phone range {} {}", startPhone, endPhone);
+ return;
+ }
+ for (int i = startPhone; i <= endPhone; i++) {
+ registerSinglePhone(i);
+ }
+ } catch (NumberFormatException nfe) {
+ LOG.warn("Invalid phone range <{},{}>", startPhoneStr, endPhoneStr);
+ }
+ }
+
+ private void registerSinglePhone(int phone)
+ {
+ phoneRegister.add(phone);
+ LOG.debug("Registered query id with phone {}", phone);
+ emitQueryResult(phone);
+ }
+
+ private void deregisterPhone(String phoneStr)
+ {
+ if (Strings.isNullOrEmpty(phoneStr)) {
+ return;
+ }
+ try {
+ Integer phone = new Integer(phoneStr);
+ // remove the channel
+ if (phoneRegister.contains(phone)) {
+ phoneRegister.remove(phone);
+ LOG.debug("Removing query id {}", phone);
+ emitPhoneRemoved(phone);
+ }
+ } catch (NumberFormatException nfe) {
+ LOG.warn("Invalid phone {}", phoneStr);
+ }
+ }
+
+ private void clearPhones()
+ {
+ phoneRegister.clear();
+ LOG.info("Clearing phones");
+ }
+
+ public final transient DefaultOutputPort<Map<String, String>> locationQueryResult = new DefaultOutputPort<Map<String, String>>();
+
+ @Override
+ public void setup(OperatorContext context)
+ {
+ this.context = context;
+ commandCounters.setCounter(CommandCounters.ADD, new MutableLong());
+ commandCounters.setCounter(CommandCounters.ADD_RANGE, new MutableLong());
+ commandCounters.setCounter(CommandCounters.DELETE, new MutableLong());
+ commandCounters.setCounter(CommandCounters.CLEAR, new MutableLong());
+ }
+
+ /**
+ * Emit all the data and clear the hash
+ */
+ @Override
+ public void endWindow()
+ {
+ for (Map.Entry<Integer, HighLow<Integer>> e: newgps.entrySet()) {
+ HighLow<Integer> loc = gps.get(e.getKey());
+ if (loc == null) {
+ gps.put(e.getKey(), e.getValue());
+ } else {
+ loc.setHigh(e.getValue().getHigh());
+ loc.setLow(e.getValue().getLow());
+ }
+ }
+ boolean found = false;
+ for (Integer phone: phoneRegister) {
+ emitQueryResult( phone);
+ found = true;
+ }
+ if (!found) {
+ LOG.debug("No phone number");
+ }
+ newgps.clear();
+ context.setCounters(commandCounters);
+ }
+
+ private void emitQueryResult(Integer phone)
+ {
+ HighLow<Integer> loc = gps.get(phone);
+ if (loc != null) {
+ Map<String, String> queryResult = new HashMap<String, String>();
+ queryResult.put(KEY_PHONE, String.valueOf(phone));
+ queryResult.put(KEY_LOCATION, loc.toString());
+ locationQueryResult.emit(queryResult);
+ }
+ }
+
+ private void emitPhoneRemoved(Integer phone)
+ {
+ Map<String,String> removedResult = Maps.newHashMap();
+ removedResult.put(KEY_PHONE, String.valueOf(phone));
+ removedResult.put(KEY_REMOVED,"true");
+ locationQueryResult.emit(removedResult);
+ }
+
+ public static enum CommandCounters
+ {
+ ADD, ADD_RANGE, DELETE, CLEAR
+ }
+
+ private static final Logger LOG = LoggerFactory.getLogger(PhoneMovementGenerator.class);
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/mobile/src/main/java/org/apache/apex/examples/mobile/doc-files/Mobile.png
----------------------------------------------------------------------
diff --git a/examples/mobile/src/main/java/org/apache/apex/examples/mobile/doc-files/Mobile.png b/examples/mobile/src/main/java/org/apache/apex/examples/mobile/doc-files/Mobile.png
new file mode 100644
index 0000000..a25da0d
Binary files /dev/null and b/examples/mobile/src/main/java/org/apache/apex/examples/mobile/doc-files/Mobile.png differ
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/mobile/src/main/java/org/apache/apex/examples/mobile/package-info.java
----------------------------------------------------------------------
diff --git a/examples/mobile/src/main/java/org/apache/apex/examples/mobile/package-info.java b/examples/mobile/src/main/java/org/apache/apex/examples/mobile/package-info.java
new file mode 100644
index 0000000..1262504
--- /dev/null
+++ b/examples/mobile/src/main/java/org/apache/apex/examples/mobile/package-info.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/**
+ * Mobile phones tracking demonstration application.
+ */
+package org.apache.apex.examples.mobile;
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/mobile/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/examples/mobile/src/main/resources/META-INF/properties.xml b/examples/mobile/src/main/resources/META-INF/properties.xml
new file mode 100644
index 0000000..e213d18
--- /dev/null
+++ b/examples/mobile/src/main/resources/META-INF/properties.xml
@@ -0,0 +1,82 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<configuration>
+ <property>
+ <name>dt.application.MobileExample.coolDownMillis</name>
+ <value>45000</value>
+ </property>
+ <property>
+ <name>dt.application.MobileExample.maxThroughput</name>
+ <value>30000</value>
+ </property>
+ <property>
+ <name>dt.application.MobileExample.minThroughput</name>
+ <value>10000</value>
+ </property>
+ <property>
+ <name>dt.application.MobileExample.operator.Receiver.tuplesBlast</name>
+ <value>200</value>
+ </property>
+ <property>
+ <name>dt.application.MobileExample.operator.Receiver.tuplesBlastIntervalMillis</name>
+ <value>5</value>
+ </property>
+ <property>
+ <name>dt.application.MobileExample.operator.Receiver.outputport.integer_data.attr.QUEUE_CAPACITY</name>
+ <value>32768</value>
+ </property>
+ <property>
+ <name>dt.application.MobileExample.operator.LocationFinder.range</name>
+ <value>20</value>
+ </property>
+ <property>
+ <name>dt.application.MobileExample.operator.LocationFinder.threshold</name>
+ <value>80</value>
+ </property>
+ <property>
+ <name>dt.application.MobileExample.operator.LocationFinder.inputport.data.attr.QUEUE_CAPACITY</name>
+ <value>32768</value>
+ </property>
+ <property>
+ <name>dt.application.MobileExample.operator.LocationResults.prop.topic</name>
+ <value>examples.mobile.phoneLocationQueryResult</value>
+ </property>
+ <property>
+ <name>dt.application.MobileExample.operator.QueryLocation.prop.topic</name>
+ <value>examples.mobile.phoneLocationQuery</value>
+ </property>
+ <property>
+ <name>dt.application.MobileExample.operator.*.attr.MEMORY_MB</name>
+ <value>768</value>
+ </property>
+ <property>
+ <name>dt.application.MobileExample.operator.*.attr.JVM_OPTIONS</name>
+ <value>-Xmx128m</value>
+ </property>
+ <property>
+ <name>dt.application.MobileExample.operator.*.port.*.attr.BUFFER_MEMORY_MB</name>
+ <value>256</value>
+ </property>
+ <property>
+ <name>dt.application.MobileExample.attr.MASTER_MEMORY_MB</name>
+ <value>1024</value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/mobile/src/test/java/org/apache/apex/examples/mobile/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/mobile/src/test/java/org/apache/apex/examples/mobile/ApplicationTest.java b/examples/mobile/src/test/java/org/apache/apex/examples/mobile/ApplicationTest.java
new file mode 100644
index 0000000..ce6ca41
--- /dev/null
+++ b/examples/mobile/src/test/java/org/apache/apex/examples/mobile/ApplicationTest.java
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.mobile;
+
+import java.net.URI;
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.servlet.Servlet;
+
+import org.eclipse.jetty.server.Connector;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.servlet.ServletContextHandler;
+import org.eclipse.jetty.servlet.ServletHolder;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.LocalMode;
+
+import com.datatorrent.lib.helper.SamplePubSubWebSocketServlet;
+import com.datatorrent.lib.io.PubSubWebSocketInputOperator;
+import com.datatorrent.lib.io.PubSubWebSocketOutputOperator;
+import com.datatorrent.lib.testbench.CollectorTestSink;
+
+public class ApplicationTest
+{
+ private static final Logger LOG = LoggerFactory.getLogger(ApplicationTest.class);
+
+ public ApplicationTest()
+ {
+ }
+
+ /**
+ * Test of getApplication method, of class Application.
+ */
+ @Test
+ public void testGetApplication() throws Exception
+ {
+ Configuration conf = new Configuration(false);
+ conf.addResource("dt-site-mobile.xml");
+ Server server = new Server(0);
+ Servlet servlet = new SamplePubSubWebSocketServlet();
+ ServletHolder sh = new ServletHolder(servlet);
+ ServletContextHandler contextHandler = new ServletContextHandler(server, "/", ServletContextHandler.SESSIONS);
+ contextHandler.addServlet(sh, "/pubsub");
+ contextHandler.addServlet(sh, "/*");
+ server.start();
+ Connector[] connector = server.getConnectors();
+ conf.set("dt.attr.GATEWAY_CONNECT_ADDRESS", "localhost:" + connector[0].getLocalPort());
+ URI uri = URI.create("ws://localhost:" + connector[0].getLocalPort() + "/pubsub");
+
+ PubSubWebSocketOutputOperator<Object> outputOperator = new PubSubWebSocketOutputOperator<Object>();
+ outputOperator.setUri(uri);
+ outputOperator.setTopic(conf.get("dt.application.MobileExample.operator.QueryLocation.topic"));
+
+ PubSubWebSocketInputOperator<Map<String, String>> inputOperator = new PubSubWebSocketInputOperator<Map<String, String>>();
+ inputOperator.setUri(uri);
+ inputOperator.setTopic(conf.get("dt.application.MobileExample.operator.LocationResults.topic"));
+
+ CollectorTestSink<Object> sink = new CollectorTestSink<Object>();
+ inputOperator.outputPort.setSink(sink);
+
+ Map<String, String> data = new HashMap<String, String>();
+ data.put("command", "add");
+ data.put("phone", "5559990");
+
+ Application app = new Application();
+ LocalMode lma = LocalMode.newInstance();
+ lma.prepareDAG(app, conf);
+ LocalMode.Controller lc = lma.getController();
+ lc.setHeartbeatMonitoringEnabled(false);
+ lc.runAsync();
+ Thread.sleep(5000);
+ inputOperator.setup(null);
+ outputOperator.setup(null);
+ inputOperator.activate(null);
+ outputOperator.beginWindow(0);
+ outputOperator.input.process(data);
+ outputOperator.endWindow();
+ inputOperator.beginWindow(0);
+ int timeoutMillis = 5000;
+ while (sink.collectedTuples.size() < 5 && timeoutMillis > 0) {
+ inputOperator.emitTuples();
+ timeoutMillis -= 20;
+ Thread.sleep(20);
+ }
+ inputOperator.endWindow();
+ lc.shutdown();
+ inputOperator.teardown();
+ outputOperator.teardown();
+ server.stop();
+ Assert.assertTrue("size of output is 5 ", sink.collectedTuples.size() == 5);
+ for (Object obj : sink.collectedTuples) {
+ Assert.assertEquals("Expected phone number", "5559990", ((Map<String, String>)obj).get("phone"));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/mobile/src/test/resources/dt-site-mobile.xml
----------------------------------------------------------------------
diff --git a/examples/mobile/src/test/resources/dt-site-mobile.xml b/examples/mobile/src/test/resources/dt-site-mobile.xml
new file mode 100644
index 0000000..dcb6c98
--- /dev/null
+++ b/examples/mobile/src/test/resources/dt-site-mobile.xml
@@ -0,0 +1,87 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<configuration>
+ <property>
+ <name>dt.application.MobileExample.class</name>
+ <value>org.apache.apex.examples.mobile.Application</value>
+ <description>An alias for the application</description>
+ </property>
+ <!--property>
+ <name>dt.attr.GATEWAY_CONNECT_ADDRESS</name>
+ <value>localhost:19090</value>
+ </property-->
+ <property>
+ <name>dt.application.MobileExample.totalSeedNumbers</name>
+ <value>0</value>
+ </property>
+ <property>
+ <name>dt.application.MobileExample.coolDownMillis</name>
+ <value>45000</value>
+ </property>
+ <property>
+ <name>dt.application.MobileExample.maxThroughput</name>
+ <value>30000</value>
+ </property>
+ <property>
+ <name>dt.application.MobileExample.minThroughput</name>
+ <value>1</value>
+ </property>
+ <property>
+ <name>dt.application.MobileExample.operator.Receiver.tuplesBlast</name>
+ <value>200</value>
+ </property>
+ <property>
+ <name>dt.application.MobileExample.operator.Receiver.tuplesBlastIntervalMillis</name>
+ <value>5</value>
+ </property>
+ <property>
+ <name>dt.application.MobileExample.operator.Receiver.outputport.integer_data.attr.QUEUE_CAPACITY</name>
+ <value>32768</value>
+ </property>
+ <property>
+ <name>dt.application.MobileExample.operator.LocationFinder.range</name>
+ <value>20</value>
+ </property>
+ <property>
+ <name>dt.application.MobileExample.operator.LocationFinder.threshold</name>
+ <value>80</value>
+ </property>
+ <property>
+ <name>dt.application.MobileExample.operator.LocationFinder.inputport.data.attr.QUEUE_CAPACITY</name>
+ <value>32768</value>
+ </property>
+ <property>
+ <name>dt.application.MobileExample.operator.LocationResults.topic</name>
+ <value>resultTopic</value>
+ </property>
+ <property>
+ <name>dt.application.MobileExample.operator.QueryLocation.topic</name>
+ <value>queryTopic</value>
+ </property>
+ <property>
+ <name>dt.application.MobileExample.operator.*.attr.MEMORY_MB</name>
+ <value>2048</value>
+ </property>
+ <property>
+ <name>dt.application.MobileExample.attr.MASTER_MEMORY_MB</name>
+ <value>1024</value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/mobile/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/examples/mobile/src/test/resources/log4j.properties b/examples/mobile/src/test/resources/log4j.properties
new file mode 100644
index 0000000..cf0d19e
--- /dev/null
+++ b/examples/mobile/src/test/resources/log4j.properties
@@ -0,0 +1,43 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+log4j.rootLogger=DEBUG,CONSOLE
+
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+log4j.appender.CONSOLE.threshold=${test.log.console.threshold}
+test.log.console.threshold=DEBUG
+
+log4j.appender.RFA=org.apache.log4j.RollingFileAppender
+log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
+log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+log4j.appender.RFA.File=/tmp/app.log
+
+# to enable, add SYSLOG to rootLogger
+log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender
+log4j.appender.SYSLOG.syslogHost=127.0.0.1
+log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout
+log4j.appender.SYSLOG.layout.conversionPattern=${dt.cid} %-5p [%t] %c{2} %x - %m%n
+log4j.appender.SYSLOG.Facility=LOCAL1
+
+log4j.logger.org=info
+#log4j.logger.org.apache.commons.beanutils=warn
+log4j.logger.com.datatorrent=debug
+log4j.logger.org.apache.apex=debug
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/mrmonitor/pom.xml
----------------------------------------------------------------------
diff --git a/examples/mrmonitor/pom.xml b/examples/mrmonitor/pom.xml
new file mode 100644
index 0000000..527565b
--- /dev/null
+++ b/examples/mrmonitor/pom.xml
@@ -0,0 +1,64 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>malhar-examples-mrmonitor</artifactId>
+ <packaging>jar</packaging>
+
+ <name>Apache Apex Malhar MR Monitoring Example</name>
+ <description></description>
+
+ <parent>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-examples</artifactId>
+ <version>3.7.0-SNAPSHOT</version>
+ </parent>
+
+ <properties>
+ <skipTests>true</skipTests>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-websocket</artifactId>
+ <version>8.1.10.v20130312</version>
+ <scope>test</scope>
+ <type>jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-servlet</artifactId>
+ <version>8.1.10.v20130312</version>
+ <scope>test</scope>
+ <type>jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>4.3.5</version>
+ <type>jar</type>
+ </dependency>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/mrmonitor/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/examples/mrmonitor/src/assemble/appPackage.xml b/examples/mrmonitor/src/assemble/appPackage.xml
new file mode 100644
index 0000000..4138cf2
--- /dev/null
+++ b/examples/mrmonitor/src/assemble/appPackage.xml
@@ -0,0 +1,59 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+ <id>appPackage</id>
+ <formats>
+ <format>jar</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>${basedir}/target/</directory>
+ <outputDirectory>/app</outputDirectory>
+ <includes>
+ <include>${project.artifactId}-${project.version}.jar</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/target/deps</directory>
+ <outputDirectory>/lib</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/site/conf</directory>
+ <outputDirectory>/conf</outputDirectory>
+ <includes>
+ <include>*.xml</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/META-INF</directory>
+ <outputDirectory>/META-INF</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/app</directory>
+ <outputDirectory>/app</outputDirectory>
+ </fileSet>
+ </fileSets>
+
+</assembly>
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/mrmonitor/src/main/java/org/apache/apex/examples/mrmonitor/Application.java
----------------------------------------------------------------------
diff --git a/examples/mrmonitor/src/main/java/org/apache/apex/examples/mrmonitor/Application.java b/examples/mrmonitor/src/main/java/org/apache/apex/examples/mrmonitor/Application.java
new file mode 100644
index 0000000..0557919
--- /dev/null
+++ b/examples/mrmonitor/src/main/java/org/apache/apex/examples/mrmonitor/Application.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.mrmonitor;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.DAG.Locality;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.lib.io.ConsoleOutputOperator;
+import com.datatorrent.lib.testbench.SeedEventGenerator;
+
+/**
+ * Application
+ *
+ * @since 2.0.0
+ */
+@ApplicationAnnotation(name = "MyFirstApplication")
+public class Application implements StreamingApplication
+{
+
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ // Sample DAG with 2 operators
+ // Replace this code with the DAG you want to build
+
+ SeedEventGenerator seedGen = dag.addOperator("seedGen", SeedEventGenerator.class);
+ seedGen.setSeedStart(1);
+ seedGen.setSeedEnd(10);
+ seedGen.addKeyData("x", 0, 10);
+ seedGen.addKeyData("y", 0, 100);
+
+ ConsoleOutputOperator cons = dag.addOperator("console", new ConsoleOutputOperator());
+ cons.setStringFormat("hello: %s");
+
+ dag.addStream("seeddata", seedGen.val_list, cons.input).setLocality(Locality.CONTAINER_LOCAL);
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/mrmonitor/src/main/java/org/apache/apex/examples/mrmonitor/Constants.java
----------------------------------------------------------------------
diff --git a/examples/mrmonitor/src/main/java/org/apache/apex/examples/mrmonitor/Constants.java b/examples/mrmonitor/src/main/java/org/apache/apex/examples/mrmonitor/Constants.java
new file mode 100644
index 0000000..1ed1f26
--- /dev/null
+++ b/examples/mrmonitor/src/main/java/org/apache/apex/examples/mrmonitor/Constants.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.mrmonitor;
+
+/**
+ * <p>Constants class.</p>
+ *
+ * @since 0.3.4
+ */
+public interface Constants
+{
+
+ public static final int MAX_NUMBER_OF_JOBS = 25;
+
+ public static final String REDUCE_TASK_TYPE = "REDUCE";
+ public static final String MAP_TASK_TYPE = "MAP";
+ public static final String TASK_TYPE = "type";
+ public static final String TASK_ID = "id";
+
+ public static final String LEAGACY_TASK_ID = "taskId";
+ public static final int MAX_TASKS = 2000;
+
+ public static final String QUERY_APP_ID = "app_id";
+ public static final String QUERY_JOB_ID = "job_id";
+ public static final String QUERY_HADOOP_VERSION = "hadoop_version";
+ public static final String QUERY_API_VERSION = "api_version";
+ public static final String QUERY_RM_PORT = "rm_port";
+ public static final String QUERY_HS_PORT = "hs_port";
+ public static final String QUERY_HOST_NAME = "hostname";
+ public static final String QUERY_KEY_COMMAND = "command";
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/mrmonitor/src/main/java/org/apache/apex/examples/mrmonitor/MRJobStatusOperator.java
----------------------------------------------------------------------
diff --git a/examples/mrmonitor/src/main/java/org/apache/apex/examples/mrmonitor/MRJobStatusOperator.java b/examples/mrmonitor/src/main/java/org/apache/apex/examples/mrmonitor/MRJobStatusOperator.java
new file mode 100644
index 0000000..24b3887
--- /dev/null
+++ b/examples/mrmonitor/src/main/java/org/apache/apex/examples/mrmonitor/MRJobStatusOperator.java
@@ -0,0 +1,622 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.mrmonitor;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.apex.examples.mrmonitor.MRStatusObject.TaskObject;
+
+import com.datatorrent.api.Context.OperatorContext;
+import com.datatorrent.api.DefaultInputPort;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.Operator;
+import com.datatorrent.api.Operator.IdleTimeHandler;
+
+/**
+ * <p>
+ * MRJobStatusOperator class.
+ * </p>
+ *
+ * @since 0.3.4
+ */
+public class MRJobStatusOperator implements Operator, IdleTimeHandler
+{
+ private static final Logger LOG = LoggerFactory.getLogger(MRJobStatusOperator.class);
+
+ private static final String JOB_PREFIX = "job_";
+ /**
+ * This outputs the meta information of the job
+ */
+ public final transient DefaultOutputPort<String> output = new DefaultOutputPort<String>();
+ /**
+ * This outputs the map task information of the job
+ */
+ public final transient DefaultOutputPort<String> mapOutput = new DefaultOutputPort<String>();
+ /**
+ * This outputs the reduce task information of the job
+ */
+ public final transient DefaultOutputPort<String> reduceOutput = new DefaultOutputPort<String>();
+ /**
+ * This outputs the counter information of the job
+ */
+ public final transient DefaultOutputPort<String> counterOutput = new DefaultOutputPort<String>();
+ /**
+ * This is time in Ms before making new request for data
+ */
+ private transient int sleepTime = 100;
+ /**
+ * This is the number of consecutive windows of no change before the job is removed from map
+ */
+ private int maxRetrials = 10;
+ /**
+ * The number of minutes for which the status history of map and reduce tasks is stored
+ */
+ private int statusHistoryTime = 60;
+ private Map<String, MRStatusObject> jobMap = new HashMap<String, MRStatusObject>();
+ /**
+ * This represents the maximum number of jobs the single instance of this operator is going to server at any time
+ */
+ private int maxJobs = Constants.MAX_NUMBER_OF_JOBS;
+ private transient Iterator<MRStatusObject> iterator;
+
+ /*
+ * each input string is a map of the following format {"app_id":<>,"hadoop_version":<>,"api_version":<>,"command":<>,
+ * "hostname":<>,"hs_port":<>,"rm_port":<>,"job_id":<>}
+ */
+ public final transient DefaultInputPort<MRStatusObject> input = new DefaultInputPort<MRStatusObject>()
+ {
+ @Override
+ public void process(MRStatusObject mrStatusObj)
+ {
+
+ if (jobMap == null) {
+ jobMap = new HashMap<String, MRStatusObject>();
+ }
+
+ if (jobMap.size() >= maxJobs) {
+ return;
+ }
+
+ if ("delete".equalsIgnoreCase(mrStatusObj.getCommand())) {
+ removeJob(mrStatusObj.getJobId());
+ JSONObject outputJsonObject = new JSONObject();
+ try {
+ outputJsonObject.put("id", mrStatusObj.getJobId());
+ outputJsonObject.put("removed", "true");
+ output.emit(outputJsonObject.toString());
+ } catch (JSONException e) {
+ LOG.warn("Error creating JSON: {}", e.getMessage());
+ }
+ return;
+ }
+ if ("clear".equalsIgnoreCase(mrStatusObj.getCommand())) {
+ clearMap();
+ return;
+ }
+
+ if (jobMap.get(mrStatusObj.getJobId()) != null) {
+ mrStatusObj = jobMap.get(mrStatusObj.getJobId());
+ }
+ if (mrStatusObj.getHadoopVersion() == 2) {
+ getJsonForJob(mrStatusObj);
+ } else if (mrStatusObj.getHadoopVersion() == 1) {
+ getJsonForLegacyJob(mrStatusObj);
+ }
+ mrStatusObj.setStatusHistoryCount(statusHistoryTime);
+ iterator = jobMap.values().iterator();
+ emitHelper(mrStatusObj);
+ }
+ };
+
+ public int getStatusHistoryTime()
+ {
+ return statusHistoryTime;
+ }
+
+ public void setStatusHistoryTime(int statusHistoryTime)
+ {
+ this.statusHistoryTime = statusHistoryTime;
+ if (jobMap != null && jobMap.size() > 0) {
+ for (Entry<String, MRStatusObject> entry : jobMap.entrySet()) {
+ entry.getValue().setStatusHistoryCount(statusHistoryTime);
+ }
+ }
+
+ }
+
+ /**
+ * This method gets the latest status of the job from the Resource Manager for jobs submitted on hadoop 2.x version
+ *
+ * @param statusObj
+ */
+ private void getJsonForJob(MRStatusObject statusObj)
+ {
+
+ String url = "http://" + statusObj.getUri() + ":" + statusObj.getRmPort() + "/proxy/application_" + statusObj.getAppId() + "/ws/v1/mapreduce/jobs/job_" + statusObj.getJobId();
+ String responseBody = MRUtil.getJsonForURL(url);
+
+ JSONObject jsonObj = MRUtil.getJsonObject(responseBody);
+
+ if (jsonObj == null) {
+ url = "http://" + statusObj.getUri() + ":" + statusObj.getHistoryServerPort() + "/ws/v1/history/mapreduce/jobs/job_" + statusObj.getJobId();
+ responseBody = MRUtil.getJsonForURL(url);
+ jsonObj = MRUtil.getJsonObject(responseBody);
+ }
+
+ if (jsonObj != null) {
+ if (jobMap.get(statusObj.getJobId()) != null) {
+ MRStatusObject tempObj = jobMap.get(statusObj.getJobId());
+ if (tempObj.getJsonObject().toString().equals(jsonObj.toString())) {
+ getJsonsForTasks(statusObj);
+ getCounterInfoForJob(statusObj);
+ return;
+ }
+ }
+ statusObj.setModified(true);
+ statusObj.setJsonObject(jsonObj);
+ getCounterInfoForJob(statusObj);
+ getJsonsForTasks(statusObj);
+ jobMap.put(statusObj.getJobId(), statusObj);
+ }
+ }
+
+ /**
+ * This method is used to collect the metric information about the job
+ *
+ * @param statusObj
+ */
+ private void getCounterInfoForJob(MRStatusObject statusObj)
+ {
+ String url = "http://" + statusObj.getUri() + ":" + statusObj.getRmPort() + "/proxy/application_" + statusObj.getAppId() + "/ws/v1/mapreduce/jobs/job_" + statusObj.getJobId() + "/counters";
+ String responseBody = MRUtil.getJsonForURL(url);
+ JSONObject jsonObj = MRUtil.getJsonObject(responseBody);
+ if (jsonObj == null) {
+ url = "http://" + statusObj.getUri() + ":" + statusObj.getHistoryServerPort() + "/ws/v1/history/mapreduce/jobs/job_" + statusObj.getJobId() + "/counters";
+ responseBody = MRUtil.getJsonForURL(url);
+ jsonObj = MRUtil.getJsonObject(responseBody);
+ }
+
+ if (jsonObj != null) {
+ if (statusObj.getMetricObject() == null) {
+ statusObj.setMetricObject(new TaskObject(jsonObj));
+ } else if (!statusObj.getMetricObject().getJsonString().equalsIgnoreCase(jsonObj.toString())) {
+ statusObj.getMetricObject().setJson(jsonObj);
+ statusObj.getMetricObject().setModified(true);
+ }
+ }
+ }
+
+ /**
+ * This method gets the latest status of the tasks for a job from the Resource Manager for jobs submitted on hadoop
+ * 2.x version
+ *
+ * @param statusObj
+ */
+ private void getJsonsForTasks(MRStatusObject statusObj)
+ {
+ String url = "http://" + statusObj.getUri() + ":" + statusObj.getRmPort() + "/proxy/application_" + statusObj.getAppId() + "/ws/v1/mapreduce/jobs/job_" + statusObj.getJobId() + "/tasks/";
+ String responseBody = MRUtil.getJsonForURL(url);
+
+ JSONObject jsonObj = MRUtil.getJsonObject(responseBody);
+ if (jsonObj == null) {
+ url = "http://" + statusObj.getUri() + ":" + statusObj.getHistoryServerPort() + "/ws/v1/history/mapreduce/jobs/job_" + statusObj.getJobId() + "/tasks/";
+ responseBody = MRUtil.getJsonForURL(url);
+
+ jsonObj = MRUtil.getJsonObject(responseBody);
+ }
+
+ if (jsonObj != null) {
+
+ try {
+ Map<String, TaskObject> mapTaskOject = statusObj.getMapJsonObject();
+ Map<String, TaskObject> reduceTaskOject = statusObj.getReduceJsonObject();
+ JSONArray taskJsonArray = jsonObj.getJSONObject("tasks").getJSONArray("task");
+
+ for (int i = 0; i < taskJsonArray.length(); i++) {
+ JSONObject taskObj = taskJsonArray.getJSONObject(i);
+ if (Constants.REDUCE_TASK_TYPE.equalsIgnoreCase(taskObj.getString(Constants.TASK_TYPE))) {
+ if (reduceTaskOject.get(taskObj.getString(Constants.TASK_ID)) != null) {
+ TaskObject tempTaskObj = reduceTaskOject.get(taskObj.getString(Constants.TASK_ID));
+ if (tempTaskObj.getJsonString().equals(taskObj.toString())) {
+ continue;
+ }
+ tempTaskObj.setJson(taskObj);
+ tempTaskObj.setModified(true);
+ reduceTaskOject.put(taskObj.getString(Constants.TASK_ID), tempTaskObj);
+ continue;
+ }
+ reduceTaskOject.put(taskObj.getString(Constants.TASK_ID), new TaskObject(taskObj));
+ } else {
+ if (mapTaskOject.get(taskObj.getString(Constants.TASK_ID)) != null) {
+ TaskObject tempTaskObj = mapTaskOject.get(taskObj.getString(Constants.TASK_ID));
+ if (tempTaskObj.getJsonString().equals(taskObj.toString())) {
+ continue;
+ }
+ tempTaskObj.setJson(taskObj);
+ tempTaskObj.setModified(true);
+ mapTaskOject.put(taskObj.getString(Constants.TASK_ID), tempTaskObj);
+ continue;
+ }
+ mapTaskOject.put(taskObj.getString(Constants.TASK_ID), new TaskObject(taskObj));
+ }
+ }
+ statusObj.setMapJsonObject(mapTaskOject);
+ statusObj.setReduceJsonObject(reduceTaskOject);
+ } catch (Exception e) {
+ LOG.info("exception: {}", e.getMessage());
+ }
+ }
+
+ }
+
+ /**
+ * This method gets the latest status of the job from the Task Manager for jobs submitted on hadoop 1.x version
+ *
+ * @param statusObj
+ */
+ private void getJsonForLegacyJob(MRStatusObject statusObj)
+ {
+
+ String url = "http://" + statusObj.getUri() + ":" + statusObj.getRmPort() + "/jobdetails.jsp?format=json&jobid=job_" + statusObj.getJobId();
+ String responseBody = MRUtil.getJsonForURL(url);
+
+ JSONObject jsonObj = MRUtil.getJsonObject(responseBody);
+ if (jsonObj == null) {
+ return;
+ }
+
+ if (jobMap.get(statusObj.getJobId()) != null) {
+ MRStatusObject tempObj = jobMap.get(statusObj.getJobId());
+ if (tempObj.getJsonObject().toString().equals(jsonObj.toString())) {
+ getJsonsForLegacyTasks(statusObj, "map");
+ getJsonsForLegacyTasks(statusObj, "reduce");
+ // output.emit(jsonObj.toString());
+ // removeJob(statusObj.getJobId());
+ return;
+ }
+ }
+
+ // output.emit(jsonObj.toString());
+ statusObj.setModified(true);
+ statusObj.setJsonObject(jsonObj);
+ getJsonsForLegacyTasks(statusObj, "map");
+ getJsonsForLegacyTasks(statusObj, "reduce");
+ jobMap.put(statusObj.getJobId(), statusObj);
+
+ }
+
+ /**
+ * This method gets the latest status of the tasks for a job from the Task Manager for jobs submitted on hadoop 1.x
+ * version
+ *
+ * @param statusObj
+ * @param type
+ */
+ private void getJsonsForLegacyTasks(MRStatusObject statusObj, String type)
+ {
+ try {
+ JSONObject jobJson = statusObj.getJsonObject();
+ int totalTasks = ((JSONObject)((JSONObject)jobJson.get(type + "TaskSummary")).get("taskStats")).getInt("numTotalTasks");
+ Map<String, TaskObject> taskMap;
+ if (type.equalsIgnoreCase("map")) {
+ taskMap = statusObj.getMapJsonObject();
+ } else {
+ taskMap = statusObj.getReduceJsonObject();
+ }
+
+ int totalPagenums = (totalTasks / Constants.MAX_TASKS) + 1;
+ String baseUrl = "http://" + statusObj.getUri() + ":" + statusObj.getRmPort() + "/jobtasks.jsp?type=" + type + "&format=json&jobid=job_" + statusObj.getJobId() + "&pagenum=";
+
+ for (int pagenum = 1; pagenum <= totalPagenums; pagenum++) {
+
+ String url = baseUrl + pagenum;
+ String responseBody = MRUtil.getJsonForURL(url);
+
+ JSONObject jsonObj = MRUtil.getJsonObject(responseBody);
+ if (jsonObj == null) {
+ return;
+ }
+
+ JSONArray taskJsonArray = jsonObj.getJSONArray("tasksInfo");
+
+ for (int i = 0; i < taskJsonArray.length(); i++) {
+ JSONObject taskObj = taskJsonArray.getJSONObject(i);
+ {
+ if (taskMap.get(taskObj.getString(Constants.LEAGACY_TASK_ID)) != null) {
+ TaskObject tempReduceObj = taskMap.get(taskObj.getString(Constants.LEAGACY_TASK_ID));
+ if (tempReduceObj.getJsonString().equals(taskObj.toString())) {
+ // tempReduceObj.setModified(false);
+ // taskMap.put(taskObj.getString(Constants.TASK_ID), tempReduceObj);
+ continue;
+ }
+ tempReduceObj.setJson(taskObj);
+ tempReduceObj.setModified(true);
+ taskMap.put(taskObj.getString(Constants.TASK_ID), tempReduceObj);
+ continue;
+
+ }
+ taskMap.put(taskObj.getString(Constants.LEAGACY_TASK_ID), new TaskObject(taskObj));
+ }
+ }
+ }
+
+ if (type.equalsIgnoreCase("map")) {
+ statusObj.setMapJsonObject(taskMap);
+ } else {
+ statusObj.setReduceJsonObject(taskMap);
+ }
+ } catch (Exception e) {
+ LOG.info(e.getMessage());
+ }
+
+ }
+
+ @Override
+ public void handleIdleTime()
+ {
+ try {
+ Thread.sleep(sleepTime);//
+ } catch (InterruptedException ie) {
+ // If this thread was intrrupted by nother thread
+ }
+ if (!iterator.hasNext()) {
+ iterator = jobMap.values().iterator();
+ }
+
+ if (iterator.hasNext()) {
+ MRStatusObject obj = iterator.next();
+ if (obj.getHadoopVersion() == 2) {
+ getJsonForJob(obj);
+ } else if (obj.getHadoopVersion() == 1) {
+ getJsonForLegacyJob(obj);
+ }
+ }
+ }
+
+ @Override
+ public void setup(OperatorContext context)
+ {
+ iterator = jobMap.values().iterator();
+ sleepTime = context.getValue(OperatorContext.SPIN_MILLIS);
+ }
+
+ @Override
+ public void teardown()
+ {
+ }
+
+ @Override
+ public void beginWindow(long arg0)
+ {
+ }
+
+ private void emitHelper(MRStatusObject obj)
+ {
+ try {
+ obj.setModified(false);
+ output.emit(obj.getJsonObject().toString());
+ JSONObject outputJsonObject = new JSONObject();
+
+ outputJsonObject.put("id", JOB_PREFIX + obj.getJobId());
+ outputJsonObject.put("mapHistory", new JSONArray(obj.getMapStatusHistory()));
+ outputJsonObject.put("reduceHistory", new JSONArray(obj.getReduceStatusHistory()));
+ outputJsonObject.put("physicalMemoryHistory", new JSONArray(obj.getPhysicalMemeoryStatusHistory()));
+ outputJsonObject.put("virtualMemoryHistory", new JSONArray(obj.getVirtualMemoryStatusHistory()));
+ outputJsonObject.put("cpuHistory", new JSONArray(obj.getCpuStatusHistory()));
+ output.emit(outputJsonObject.toString());
+ obj.setChangedHistoryStatus(false);
+
+ outputJsonObject = new JSONObject();
+ outputJsonObject.put("id", JOB_PREFIX + obj.getJobId());
+ JSONArray arr = new JSONArray();
+
+ for (Map.Entry<String, TaskObject> mapEntry : obj.getMapJsonObject().entrySet()) {
+ TaskObject json = mapEntry.getValue();
+ json.setModified(false);
+ arr.put(json.getJson());
+ }
+
+ outputJsonObject.put("tasks", arr);
+ mapOutput.emit(outputJsonObject.toString());
+
+ outputJsonObject = new JSONObject();
+ outputJsonObject.put("id", JOB_PREFIX + obj.getJobId());
+ arr = new JSONArray();
+
+ for (Map.Entry<String, TaskObject> mapEntry : obj.getReduceJsonObject().entrySet()) {
+ TaskObject json = mapEntry.getValue();
+ json.setModified(false);
+ arr.put(json.getJson());
+ }
+
+ outputJsonObject.put("tasks", arr);
+ reduceOutput.emit(outputJsonObject.toString());
+ obj.setRetrials(0);
+ } catch (Exception e) {
+ LOG.warn("error creating json {}", e.getMessage());
+ }
+
+ }
+
+ @Override
+ public void endWindow()
+ {
+ List<String> delList = new ArrayList<String>();
+ try {
+ for (Map.Entry<String, MRStatusObject> entry : jobMap.entrySet()) {
+ MRStatusObject obj = entry.getValue();
+
+ JSONObject outputJsonObject = new JSONObject();
+ outputJsonObject.put("id", JOB_PREFIX + obj.getJobId());
+
+ boolean modified = false;
+
+ if (obj.isModified()) {
+ modified = true;
+ obj.setModified(false);
+ output.emit(obj.getJsonObject().toString());
+ if (obj.isChangedHistoryStatus()) {
+ outputJsonObject.put("mapHistory", new JSONArray(obj.getMapStatusHistory()));
+ outputJsonObject.put("reduceHistory", new JSONArray(obj.getReduceStatusHistory()));
+ outputJsonObject.put("physicalMemoryHistory", new JSONArray(obj.getPhysicalMemeoryStatusHistory()));
+ outputJsonObject.put("virtualMemoryHistory", new JSONArray(obj.getVirtualMemoryStatusHistory()));
+ outputJsonObject.put("cpuHistory", new JSONArray(obj.getCpuStatusHistory()));
+ output.emit(outputJsonObject.toString());
+ obj.setChangedHistoryStatus(false);
+ }
+ }
+ outputJsonObject = new JSONObject();
+ outputJsonObject.put("id", JOB_PREFIX + obj.getJobId());
+ JSONArray arr = new JSONArray();
+
+ for (Map.Entry<String, TaskObject> mapEntry : obj.getMapJsonObject().entrySet()) {
+ TaskObject json = mapEntry.getValue();
+ if (json.isModified()) {
+ modified = true;
+ json.setModified(false);
+ arr.put(json.getJson());
+ }
+ }
+
+ if (arr.length() > 0) {
+ outputJsonObject.put("tasks", arr);
+ mapOutput.emit(outputJsonObject.toString());
+ }
+
+ outputJsonObject = new JSONObject();
+ outputJsonObject.put("id", JOB_PREFIX + obj.getJobId());
+ arr = new JSONArray();
+
+ for (Map.Entry<String, TaskObject> mapEntry : obj.getReduceJsonObject().entrySet()) {
+ TaskObject json = mapEntry.getValue();
+ if (json.isModified()) {
+ modified = true;
+ json.setModified(false);
+ arr.put(json.getJson());
+ }
+ }
+ if (arr.length() > 0) {
+ outputJsonObject.put("tasks", arr);
+ reduceOutput.emit(outputJsonObject.toString());
+ }
+
+ if (obj.getMetricObject() != null && obj.getMetricObject().isModified()) {
+ modified = true;
+ obj.getMetricObject().setModified(false);
+ counterOutput.emit(obj.getMetricObject().getJsonString());
+ }
+
+ if (!modified) {
+ if (obj.getRetrials() >= maxRetrials) {
+ delList.add(obj.getJobId());
+ } else {
+ obj.setRetrials(obj.getRetrials() + 1);
+ }
+ } else {
+ obj.setRetrials(0);
+ }
+ }
+ } catch (Exception ex) {
+ LOG.warn("error creating json {}", ex.getMessage());
+ }
+
+ if (!delList.isEmpty()) {
+ Iterator<String> itr = delList.iterator();
+ while (itr.hasNext()) {
+ removeJob(itr.next());
+ }
+ }
+
+ }
+
+ /**
+ * This method removes the job from the map
+ *
+ * @param jobId
+ */
+ public void removeJob(String jobId)
+ {
+ if (jobMap != null) {
+ jobMap.remove(jobId);
+ iterator = jobMap.values().iterator();
+ }
+ }
+
+ /**
+ * This method clears the job map
+ */
+ public void clearMap()
+ {
+ if (jobMap != null) {
+ jobMap.clear();
+ iterator = jobMap.values().iterator();
+ }
+ }
+
+ /**
+ * This returns the maximum number of jobs the single instance of this operator is going to server at any time
+ *
+ * @return
+ */
+ public int getMaxJobs()
+ {
+ return maxJobs;
+ }
+
+ /**
+ * This sets the maximum number of jobs the single instance of this operator is going to server at any time
+ *
+ * @param maxJobs
+ */
+ public void setMaxJobs(int maxJobs)
+ {
+ this.maxJobs = maxJobs;
+ }
+
+ /**
+ * This sets the number of consecutive windows of no change before the job is removed from map
+ *
+ * @return
+ */
+ public int getMaxRetrials()
+ {
+ return maxRetrials;
+ }
+
+ /**
+ * This returns the number of consecutive windows of no change before the job is removed from map
+ *
+ * @param maxRetrials
+ */
+ public void setMaxRetrials(int maxRetrials)
+ {
+ this.maxRetrials = maxRetrials;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/mrmonitor/src/main/java/org/apache/apex/examples/mrmonitor/MRMonitoringApplication.java
----------------------------------------------------------------------
diff --git a/examples/mrmonitor/src/main/java/org/apache/apex/examples/mrmonitor/MRMonitoringApplication.java b/examples/mrmonitor/src/main/java/org/apache/apex/examples/mrmonitor/MRMonitoringApplication.java
new file mode 100644
index 0000000..288da84
--- /dev/null
+++ b/examples/mrmonitor/src/main/java/org/apache/apex/examples/mrmonitor/MRMonitoringApplication.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.mrmonitor;
+
+import java.net.URI;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+
+import com.datatorrent.lib.io.PubSubWebSocketInputOperator;
+import com.datatorrent.lib.io.PubSubWebSocketOutputOperator;
+
+/**
+ * <p>
+ * MRDebuggerApplication class.
+ * </p>
+ *
+ * @since 0.3.4
+ */
+@ApplicationAnnotation(name = "MRMonitoringExample")
+public class MRMonitoringApplication implements StreamingApplication
+{
+
+ private static final Logger logger = LoggerFactory.getLogger(MRMonitoringApplication.class);
+
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ String daemonAddress = dag.getValue(DAG.GATEWAY_CONNECT_ADDRESS);
+ MRJobStatusOperator mrJobOperator = dag.addOperator("JobMonitor", new MRJobStatusOperator());
+ URI uri = URI.create("ws://" + daemonAddress + "/pubsub");
+ logger.info("WebSocket with daemon at {}", daemonAddress);
+
+ PubSubWebSocketInputOperator wsIn = dag.addOperator("Query", new PubSubWebSocketInputOperator());
+ wsIn.setUri(uri);
+
+ MapToMRObjectOperator queryConverter = dag.addOperator("QueryConverter", new MapToMRObjectOperator());
+
+ /**
+ * This is used to emit the meta data about the job
+ */
+ PubSubWebSocketOutputOperator<Object> wsOut = dag.addOperator("JobOutput", new PubSubWebSocketOutputOperator<Object>());
+ wsOut.setUri(uri);
+
+ /**
+ * This is used to emit the information of map tasks of the job
+ */
+ PubSubWebSocketOutputOperator<Object> wsMapOut = dag.addOperator("MapJob", new PubSubWebSocketOutputOperator<Object>());
+ wsMapOut.setUri(uri);
+
+ /**
+ * This is used to emit the information of reduce tasks of the job
+ */
+ PubSubWebSocketOutputOperator<Object> wsReduceOut = dag.addOperator("ReduceJob", new PubSubWebSocketOutputOperator<Object>());
+ wsReduceOut.setUri(uri);
+
+ /**
+ * This is used to emit the metric information of the job
+ */
+ PubSubWebSocketOutputOperator<Object> wsCounterOut = dag.addOperator("JobCounter", new PubSubWebSocketOutputOperator<Object>());
+ wsCounterOut.setUri(uri);
+
+ dag.addStream("QueryConversion", wsIn.outputPort, queryConverter.input);
+ dag.addStream("QueryProcessing", queryConverter.output, mrJobOperator.input);
+ dag.addStream("JobData", mrJobOperator.output, wsOut.input);
+ dag.addStream("MapData", mrJobOperator.mapOutput, wsMapOut.input);
+ dag.addStream("ReduceData", mrJobOperator.reduceOutput, wsReduceOut.input);
+ dag.addStream("CounterData", mrJobOperator.counterOutput, wsCounterOut.input);
+ }
+
+}
[12/30] apex-malhar git commit: Renamed demos to examples. Packages
and artifactid names are changed as suggested.
Posted by th...@apache.org.
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/AutoComplete.java
----------------------------------------------------------------------
diff --git a/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/AutoComplete.java b/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/AutoComplete.java
new file mode 100644
index 0000000..2db59b6
--- /dev/null
+++ b/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/AutoComplete.java
@@ -0,0 +1,324 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.malhar.stream.sample.complete;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.apex.malhar.lib.window.TriggerOption;
+import org.apache.apex.malhar.lib.window.Tuple;
+import org.apache.apex.malhar.lib.window.Window;
+import org.apache.apex.malhar.lib.window.WindowOption;
+import org.apache.apex.malhar.stream.api.ApexStream;
+import org.apache.apex.malhar.stream.api.CompositeStreamTransform;
+import org.apache.apex.malhar.stream.api.WindowedStream;
+import org.apache.apex.malhar.stream.api.function.Function;
+import org.apache.apex.malhar.stream.api.impl.StreamFactory;
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
+
+import com.google.common.base.Throwables;
+
+import com.datatorrent.api.Context.OperatorContext;
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.DefaultInputPort;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.InputOperator;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.common.util.BaseOperator;
+import com.datatorrent.lib.util.KeyValPair;
+
+import static org.apache.apex.malhar.stream.api.Option.Options.name;
+
+/**
+ * An example that computes the most popular hash tags
+ * for every prefix, which can be used for auto-completion.
+ * This application is identical to TwitterAutoComplete, except it's
+ * reading from a file. This application is mainly for local testing
+ * purpose.
+ *
+ * <p>This will update the datastore every 10 seconds based on the last
+ * 30 minutes of data received.
+ *
+ * @since 3.5.0
+ */
+@ApplicationAnnotation(name = "AutoComplete")
+public class AutoComplete implements StreamingApplication
+{
+
+ /**
+ * A dummy Twitter input operator. It reads from a text file containing some tweets and output a line every
+ * half of a second.
+ */
+ public static class TweetsInput extends BaseOperator implements InputOperator
+ {
+ public final transient DefaultOutputPort<String> output = new DefaultOutputPort<>();
+ private boolean done;
+
+ private transient BufferedReader reader;
+
+ @Override
+ public void setup(OperatorContext context)
+ {
+ done = false;
+ initReader();
+ }
+
+ private void initReader()
+ {
+ try {
+ InputStream resourceStream = this.getClass().getResourceAsStream("/sampletweets.txt");
+ reader = new BufferedReader(new InputStreamReader(resourceStream));
+ } catch (Exception ex) {
+ throw Throwables.propagate(ex);
+ }
+ }
+
+ @Override
+ public void teardown()
+ {
+ IOUtils.closeQuietly(reader);
+ }
+
+ @Override
+ public void emitTuples()
+ {
+ if (!done) {
+ try {
+ String line = reader.readLine();
+ if (line == null) {
+ done = true;
+ reader.close();
+ } else {
+ this.output.emit(line);
+ }
+ Thread.sleep(50);
+ } catch (IOException ex) {
+ throw new RuntimeException(ex);
+ } catch (InterruptedException e) {
+ // Ignore it.
+ }
+ }
+ }
+ }
+
+ public static class Collector extends BaseOperator
+ {
+ private static Map<String, List<CompletionCandidate>> result = new HashMap<>();
+ private static boolean done = false;
+
+ public static boolean isDone()
+ {
+ return done;
+ }
+
+ @Override
+ public void setup(OperatorContext context)
+ {
+ super.setup(context);
+ done = false;
+ }
+
+ public static Map<String, List<CompletionCandidate>> getResult()
+ {
+ return result;
+ }
+
+ public final transient DefaultInputPort<Tuple.WindowedTuple<KeyValPair<String, List<CompletionCandidate>>>> input = new DefaultInputPort<Tuple.WindowedTuple<KeyValPair<String, List<CompletionCandidate>>>>()
+ {
+ @Override
+ public void process(Tuple.WindowedTuple<KeyValPair<String, List<CompletionCandidate>>> tuple)
+ {
+ if (tuple.getValue().getKey().equals("yarn")) {
+ done = true;
+ }
+ result.put(tuple.getValue().getKey(), tuple.getValue().getValue());
+ }
+ };
+ }
+
+ /**
+ * FlapMap Function to extract all hashtags from a text form tweet.
+ */
+ private static class ExtractHashtags implements Function.FlatMapFunction<String, String>
+ {
+
+ @Override
+ public Iterable<String> f(String input)
+ {
+ List<String> result = new LinkedList<>();
+ Matcher m = Pattern.compile("#\\S+").matcher(input);
+ while (m.find()) {
+ result.add(m.group().substring(1));
+ }
+ return result;
+ }
+ }
+
+ /**
+ * Lower latency, but more expensive.
+ */
+ private static class ComputeTopFlat
+ extends CompositeStreamTransform<WindowedStream<CompletionCandidate>, WindowedStream<Tuple.WindowedTuple<KeyValPair<String, List<CompletionCandidate>>>>>
+ {
+ private final int candidatesPerPrefix;
+ private final int minPrefix;
+
+ public ComputeTopFlat(int candidatesPerPrefix, int minPrefix)
+ {
+ this.candidatesPerPrefix = candidatesPerPrefix;
+ this.minPrefix = minPrefix;
+ }
+
+ @Override
+ public WindowedStream<Tuple.WindowedTuple<KeyValPair<String, List<CompletionCandidate>>>> compose(
+ WindowedStream<CompletionCandidate> input)
+ {
+ return input
+ .<KeyValPair<String, CompletionCandidate>, WindowedStream<KeyValPair<String, CompletionCandidate>>>flatMap(new AllPrefixes(minPrefix))
+ .accumulateByKey(new TopNByKey(), new Function.ToKeyValue<KeyValPair<String, CompletionCandidate>, String,
+ CompletionCandidate>()
+ {
+ @Override
+ public Tuple<KeyValPair<String, CompletionCandidate>> f(KeyValPair<String, CompletionCandidate> tuple)
+ {
+ // TODO: Should be removed after Auto-wrapping is supported.
+ return new Tuple.WindowedTuple<>(Window.GlobalWindow.INSTANCE, tuple);
+ }
+ });
+ }
+ }
+
+ /**
+ * FlapMap Function to extract all prefixes of the hashtag in the input CompletionCandidate, and output
+ * KeyValPairs of the prefix and the CompletionCandidate
+ */
+ private static class AllPrefixes implements Function.FlatMapFunction<CompletionCandidate, KeyValPair<String, CompletionCandidate>>
+ {
+ private final int minPrefix;
+ private final int maxPrefix;
+
+ public AllPrefixes()
+ {
+ this(0, Integer.MAX_VALUE);
+ }
+
+ public AllPrefixes(int minPrefix)
+ {
+ this(minPrefix, Integer.MAX_VALUE);
+ }
+
+ public AllPrefixes(int minPrefix, int maxPrefix)
+ {
+ this.minPrefix = minPrefix;
+ this.maxPrefix = maxPrefix;
+ }
+
+ @Override
+ public Iterable<KeyValPair<String, CompletionCandidate>> f(CompletionCandidate input)
+ {
+ List<KeyValPair<String, CompletionCandidate>> result = new LinkedList<>();
+ String word = input.getValue();
+ for (int i = minPrefix; i <= Math.min(word.length(), maxPrefix); i++) {
+
+ result.add(new KeyValPair<>(input.getValue().substring(0, i).toLowerCase(), input));
+ }
+ return result;
+ }
+ }
+
+ /**
+ * A Composite stream transform that takes as input a list of tokens and returns
+ * the most common tokens per prefix.
+ */
+ public static class ComputeTopCompletions
+ extends CompositeStreamTransform<WindowedStream<String>, WindowedStream<Tuple.WindowedTuple<KeyValPair<String, List<CompletionCandidate>>>>>
+ {
+ private final int candidatesPerPrefix;
+ private final boolean recursive;
+
+ protected ComputeTopCompletions(int candidatesPerPrefix, boolean recursive)
+ {
+ this.candidatesPerPrefix = candidatesPerPrefix;
+ this.recursive = recursive;
+ }
+
+ public static ComputeTopCompletions top(int candidatesPerPrefix, boolean recursive)
+ {
+ return new ComputeTopCompletions(candidatesPerPrefix, recursive);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public WindowedStream<Tuple.WindowedTuple<KeyValPair<String, List<CompletionCandidate>>>> compose(WindowedStream<String> inputStream)
+ {
+ ApexStream<CompletionCandidate> candidates = inputStream
+ .countByKey(new Function.ToKeyValue<String, String, Long>()
+ {
+ @Override
+ public Tuple<KeyValPair<String, Long>> f(String input)
+ {
+ return new Tuple.PlainTuple<>(new KeyValPair<>(input, 1L));
+ }
+ }, name("countByKey"))
+ .map(new Function.MapFunction<Tuple.WindowedTuple<KeyValPair<String,Long>>, CompletionCandidate>()
+ {
+ @Override
+ public CompletionCandidate f(Tuple.WindowedTuple<KeyValPair<String, Long>> input)
+ {
+ return new CompletionCandidate(input.getValue().getKey(), input.getValue().getValue());
+ }
+ }, name("ToCompletionCandidate"));
+
+ return candidates.addCompositeStreams(new ComputeTopFlat(10, 1));
+
+ }
+ }
+
+ /**
+ * Populate the dag with High-Level API.
+ * @param dag
+ * @param conf
+ */
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ TweetsInput input = new TweetsInput();
+ Collector collector = new Collector();
+
+ WindowOption windowOption = new WindowOption.GlobalWindow();
+
+ ApexStream<String> tags = StreamFactory.fromInput(input, input.output, name("tweetSampler"))
+ .flatMap(new ExtractHashtags());
+
+ tags.window(windowOption, new TriggerOption().accumulatingFiredPanes().withEarlyFiringsAtEvery(1))
+ .addCompositeStreams(ComputeTopCompletions.top(10, true)).print(name("console"))
+ .endWith(collector, collector.input, name("collector"))
+ .populateDag(dag);
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/CompletionCandidate.java
----------------------------------------------------------------------
diff --git a/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/CompletionCandidate.java b/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/CompletionCandidate.java
new file mode 100644
index 0000000..bd5c511
--- /dev/null
+++ b/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/CompletionCandidate.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.malhar.stream.sample.complete;
+
+/**
+ * Class used to store tag-count pairs in Auto Complete Example.
+ *
+ * @since 3.5.0
+ */
+public class CompletionCandidate implements Comparable<CompletionCandidate>
+{
+ private long count;
+ private String value;
+
+ public CompletionCandidate(String value, long count)
+ {
+ this.value = value;
+ this.count = count;
+ }
+
+ public long getCount()
+ {
+ return count;
+ }
+
+ public String getValue()
+ {
+ return value;
+ }
+
+ // Empty constructor required for Kryo.
+ public CompletionCandidate()
+ {
+
+ }
+
+ @Override
+ public int compareTo(CompletionCandidate o)
+ {
+ if (this.count < o.count) {
+ return -1;
+ } else if (this.count == o.count) {
+ return this.value.compareTo(o.value);
+ } else {
+ return 1;
+ }
+ }
+
+ @Override
+ public boolean equals(Object other)
+ {
+ if (other instanceof CompletionCandidate) {
+ CompletionCandidate that = (CompletionCandidate)other;
+ return this.count == that.count && this.value.equals(that.value);
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Long.valueOf(count).hashCode() ^ value.hashCode();
+ }
+
+ @Override
+ public String toString()
+ {
+ return "CompletionCandidate[" + value + ", " + count + "]";
+ }
+}
+
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/PojoEvent.java
----------------------------------------------------------------------
diff --git a/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/PojoEvent.java b/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/PojoEvent.java
new file mode 100644
index 0000000..ee15d90
--- /dev/null
+++ b/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/PojoEvent.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.malhar.stream.sample.complete;
+
+/**
+ * Tuple Class for JdbcOutput of StreamingWordExtract.
+ *
+ * @since 3.5.0
+ */
+public class PojoEvent extends Object
+{
+ private String stringValue;
+
+ @Override
+ public String toString()
+ {
+ return "PojoEvent [stringValue=" + getStringValue() + "]";
+ }
+
+ public void setStringValue(String newString)
+ {
+ this.stringValue = newString;
+ }
+
+ public String getStringValue()
+ {
+ return this.stringValue;
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/StreamingWordExtract.java
----------------------------------------------------------------------
diff --git a/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/StreamingWordExtract.java b/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/StreamingWordExtract.java
new file mode 100644
index 0000000..07f01d0
--- /dev/null
+++ b/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/StreamingWordExtract.java
@@ -0,0 +1,162 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.malhar.stream.sample.complete;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.apex.malhar.stream.api.ApexStream;
+import org.apache.apex.malhar.stream.api.Option;
+import org.apache.apex.malhar.stream.api.function.Function;
+import org.apache.apex.malhar.stream.api.impl.StreamFactory;
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.lib.db.jdbc.JdbcFieldInfo;
+import com.datatorrent.lib.db.jdbc.JdbcPOJOInsertOutputOperator;
+import com.datatorrent.lib.db.jdbc.JdbcTransactionalStore;
+
+import static java.sql.Types.VARCHAR;
+
+/**
+ * Beam StreamingWordExtract Example.
+ *
+ * @since 3.5.0
+ */
+@ApplicationAnnotation(name = "StreamingWordExtract")
+public class StreamingWordExtract implements StreamingApplication
+{
+ private static int wordCount = 0; // A counter to count number of words have been extracted.
+ private static int entriesMapped = 0; // A counter to count number of entries have been mapped.
+
+ public int getWordCount()
+ {
+ return wordCount;
+ }
+
+ public int getEntriesMapped()
+ {
+ return entriesMapped;
+ }
+
+ /**
+ * A MapFunction that tokenizes lines of text into individual words.
+ */
+ public static class ExtractWords implements Function.FlatMapFunction<String, String>
+ {
+ @Override
+ public Iterable<String> f(String input)
+ {
+ List<String> result = new ArrayList<>(Arrays.asList(input.split("[^a-zA-Z0-9']+")));
+ wordCount += result.size();
+ return result;
+ }
+ }
+
+
+ /**
+ * A MapFunction that uppercases a word.
+ */
+ public static class Uppercase implements Function.MapFunction<String, String>
+ {
+ @Override
+ public String f(String input)
+ {
+ return input.toUpperCase();
+ }
+ }
+
+
+ /**
+ * A filter function to filter out empty strings.
+ */
+ public static class EmptyStringFilter implements Function.FilterFunction<String>
+ {
+ @Override
+ public boolean f(String input)
+ {
+ return !input.isEmpty();
+ }
+ }
+
+
+ /**
+ * A map function to map the result string to a pojo entry.
+ */
+ public static class PojoMapper implements Function.MapFunction<String, Object>
+ {
+
+ @Override
+ public Object f(String input)
+ {
+ PojoEvent pojo = new PojoEvent();
+ pojo.setStringValue(input);
+ entriesMapped++;
+ return pojo;
+ }
+ }
+
+ /**
+ * Add field infos to the {@link JdbcPOJOInsertOutputOperator}.
+ */
+ private static List<JdbcFieldInfo> addFieldInfos()
+ {
+ List<JdbcFieldInfo> fieldInfos = new ArrayList<>();
+ fieldInfos.add(new JdbcFieldInfo("STRINGVALUE", "stringValue", JdbcFieldInfo.SupportType.STRING, VARCHAR));
+ return fieldInfos;
+ }
+
+ /**
+ * Populate dag with High-Level API.
+ * @param dag
+ * @param conf
+ */
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ JdbcPOJOInsertOutputOperator jdbcOutput = new JdbcPOJOInsertOutputOperator();
+ jdbcOutput.setFieldInfos(addFieldInfos());
+ JdbcTransactionalStore outputStore = new JdbcTransactionalStore();
+ jdbcOutput.setStore(outputStore);
+ jdbcOutput.setTablename("TestTable");
+
+ // Create a stream reading from a folder.
+ ApexStream<String> stream = StreamFactory.fromFolder("./src/test/resources/data");
+
+ // Extract all the words from the input line of text.
+ stream.flatMap(new ExtractWords())
+
+ // Filter out the empty strings.
+ .filter(new EmptyStringFilter())
+
+ // Change every word to uppercase.
+ .map(new Uppercase())
+
+ // Map the resulted word to a Pojo entry.
+ .map(new PojoMapper())
+
+ // Output the entries to JdbcOutput and insert them into a table.
+ .endWith(jdbcOutput, jdbcOutput.input, Option.Options.name("jdbcOutput"));
+
+ stream.populateDag(dag);
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/TopNByKey.java
----------------------------------------------------------------------
diff --git a/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/TopNByKey.java b/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/TopNByKey.java
new file mode 100644
index 0000000..937254c
--- /dev/null
+++ b/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/TopNByKey.java
@@ -0,0 +1,120 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.malhar.stream.sample.complete;
+
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.apex.malhar.lib.window.Accumulation;
+
+/**
+ * Specialized TopNByKey accumulation for AutoComplete Example.
+ *
+ * @since 3.5.0
+ */
+public class TopNByKey implements
+ Accumulation<CompletionCandidate, Map<String, Long>, List<CompletionCandidate>>
+{
+ int n = 10;
+
+ Comparator comparator;
+
+ public void setN(int n)
+ {
+ this.n = n;
+ }
+
+ public void setComparator(Comparator comparator)
+ {
+ this.comparator = comparator;
+ }
+
+ @Override
+ public Map<String, Long> defaultAccumulatedValue()
+ {
+ return new HashMap<>();
+ }
+
+ /**
+ * Accumulate the input. Update the entry in the Accumulation Map if the key of the input is existed, create a
+ * new entry otherwise.
+ * @param accumulatedValue
+ * @param input
+ * @return
+ */
+ @Override
+ public Map<String, Long> accumulate(Map<String, Long> accumulatedValue, CompletionCandidate input)
+ {
+ accumulatedValue.put(input.getValue(), input.getCount());
+ return accumulatedValue;
+ }
+
+ /**
+ * Merge two Maps together. For every key, keep the larger value in the resulted Map.
+ * @param accumulatedValue1
+ * @param accumulatedValue2
+ * @return
+ */
+ @Override
+ public Map<String, Long> merge(Map<String, Long> accumulatedValue1, Map<String, Long> accumulatedValue2)
+ {
+ for (Map.Entry<String, Long> entry : accumulatedValue2.entrySet()) {
+ if (accumulatedValue1.containsKey(entry.getKey()) && accumulatedValue1.get(entry.getKey()) > entry.getValue()) {
+ continue;
+ }
+ accumulatedValue1.put(entry.getKey(), entry.getValue());
+ }
+ return accumulatedValue1;
+ }
+
+ /**
+ * Loop through the Accumulation Map to get the top n entries based on their values, return a list containing
+ * those entries.
+ * @param accumulatedValue
+ * @return
+ */
+ @Override
+ public List<CompletionCandidate> getOutput(Map<String, Long> accumulatedValue)
+ {
+ LinkedList<CompletionCandidate> result = new LinkedList<>();
+ for (Map.Entry<String, Long> entry : accumulatedValue.entrySet()) {
+ int k = 0;
+ for (CompletionCandidate inMemory : result) {
+ if (entry.getValue() > inMemory.getCount()) {
+ break;
+ }
+ k++;
+ }
+ result.add(k, new CompletionCandidate(entry.getKey(), entry.getValue()));
+ if (result.size() > n) {
+ result.remove(result.get(result.size() - 1));
+ }
+ }
+ return result;
+ }
+
+ @Override
+ public List<CompletionCandidate> getRetraction(List<CompletionCandidate> value)
+ {
+ return new LinkedList<>();
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/TopWikipediaSessions.java
----------------------------------------------------------------------
diff --git a/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/TopWikipediaSessions.java b/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/TopWikipediaSessions.java
new file mode 100644
index 0000000..68ec733
--- /dev/null
+++ b/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/TopWikipediaSessions.java
@@ -0,0 +1,347 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.malhar.stream.sample.complete;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Random;
+import javax.annotation.Nullable;
+
+import org.joda.time.Duration;
+
+import org.apache.apex.malhar.lib.window.TriggerOption;
+import org.apache.apex.malhar.lib.window.Tuple;
+import org.apache.apex.malhar.lib.window.Window;
+import org.apache.apex.malhar.lib.window.WindowOption;
+import org.apache.apex.malhar.lib.window.accumulation.TopN;
+import org.apache.apex.malhar.stream.api.ApexStream;
+import org.apache.apex.malhar.stream.api.CompositeStreamTransform;
+import org.apache.apex.malhar.stream.api.WindowedStream;
+import org.apache.apex.malhar.stream.api.function.Function;
+import org.apache.apex.malhar.stream.api.impl.StreamFactory;
+import org.apache.hadoop.conf.Configuration;
+
+
+import com.datatorrent.api.Context;
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.DefaultInputPort;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.InputOperator;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.common.util.BaseOperator;
+import com.datatorrent.lib.util.KeyValPair;
+
+import static org.apache.apex.malhar.stream.api.Option.Options.name;
+
+/**
+ * Beam's TopWikipediaSessions Example.
+ *
+ * @since 3.5.0
+ */
+@ApplicationAnnotation(name = "TopWikipediaSessions")
+public class TopWikipediaSessions implements StreamingApplication
+{
+ /**
+ * A generator that outputs a stream of combinations of some users and some randomly generated edit time.
+ */
+ public static class SessionGen extends BaseOperator implements InputOperator
+ {
+ private String[] names = new String[]{"user1", "user2", "user3", "user4"};
+ public transient DefaultOutputPort<KeyValPair<String, Long>> output = new DefaultOutputPort<>();
+
+ private static final Duration RAND_RANGE = Duration.standardDays(365);
+ private Long minTimestamp;
+ private long sleepTime;
+ private static int tupleCount = 0;
+
+ public static int getTupleCount()
+ {
+ return tupleCount;
+ }
+
+ private String randomName(String[] names)
+ {
+ int index = new Random().nextInt(names.length);
+ return names[index];
+ }
+
+ @Override
+ public void setup(Context.OperatorContext context)
+ {
+ super.setup(context);
+ tupleCount = 0;
+ minTimestamp = System.currentTimeMillis();
+ sleepTime = context.getValue(Context.OperatorContext.SPIN_MILLIS);
+ }
+
+ @Override
+ public void emitTuples()
+ {
+ long randMillis = (long)(Math.random() * RAND_RANGE.getMillis());
+ long randomTimestamp = minTimestamp + randMillis;
+ output.emit(new KeyValPair<String, Long>(randomName(names), randomTimestamp));
+ tupleCount++;
+ try {
+ Thread.sleep(sleepTime);
+ } catch (InterruptedException e) {
+ // Ignore it.
+ }
+ }
+ }
+
+ public static class Collector extends BaseOperator
+ {
+ private final int resultSize = 5;
+ private static List<List<TempWrapper>> result = new ArrayList<>();
+
+ public static List<List<TempWrapper>> getResult()
+ {
+ return result;
+ }
+
+ public final transient DefaultInputPort<Tuple.WindowedTuple<List<TempWrapper>>> input = new DefaultInputPort<Tuple.WindowedTuple<List<TempWrapper>>>()
+ {
+ @Override
+ public void process(Tuple.WindowedTuple<List<TempWrapper>> tuple)
+ {
+ if (result.size() == resultSize) {
+ result.remove(0);
+ }
+ result.add(tuple.getValue());
+ }
+ };
+ }
+
+
+ /**
+ * Convert the upstream (user, time) combination to a timestamped tuple of user.
+ */
+ static class ExtractUserAndTimestamp implements Function.MapFunction<KeyValPair<String, Long>, Tuple.TimestampedTuple<String>>
+ {
+ @Override
+ public Tuple.TimestampedTuple<String> f(KeyValPair<String, Long> input)
+ {
+ long timestamp = input.getValue();
+ String userName = input.getKey();
+
+ // Sets the implicit timestamp field to be used in windowing.
+ return new Tuple.TimestampedTuple<>(timestamp, userName);
+
+ }
+ }
+
+ /**
+ * Computes the number of edits in each user session. A session is defined as
+ * a string of edits where each is separated from the next by less than an hour.
+ */
+ static class ComputeSessions
+ extends CompositeStreamTransform<ApexStream<Tuple.TimestampedTuple<String>>, WindowedStream<Tuple.WindowedTuple<KeyValPair<String, Long>>>>
+ {
+ @Override
+ public WindowedStream<Tuple.WindowedTuple<KeyValPair<String, Long>>> compose(ApexStream<Tuple.TimestampedTuple<String>> inputStream)
+ {
+ return inputStream
+
+ // Chuck the stream into session windows.
+ .window(new WindowOption.SessionWindows(Duration.standardHours(1)), new TriggerOption().accumulatingFiredPanes().withEarlyFiringsAtEvery(1))
+
+ // Count the number of edits for a user within one session.
+ .countByKey(new Function.ToKeyValue<Tuple.TimestampedTuple<String>, String, Long>()
+ {
+ @Override
+ public Tuple.TimestampedTuple<KeyValPair<String, Long>> f(Tuple.TimestampedTuple<String> input)
+ {
+ return new Tuple.TimestampedTuple<KeyValPair<String, Long>>(input.getTimestamp(), new KeyValPair<String, Long>(input.getValue(), 1L));
+ }
+ }, name("ComputeSessions"));
+ }
+ }
+
+ /**
+ * A comparator class used for comparing two TempWrapper objects.
+ */
+ public static class Comp implements Comparator<TempWrapper>
+ {
+ @Override
+ public int compare(TempWrapper o1, TempWrapper o2)
+ {
+ return Long.compare(o1.getValue().getValue(), o2.getValue().getValue());
+ }
+ }
+
+ /**
+ * A function to extract timestamp from a TempWrapper object.
+ */
+ // TODO: Need to revisit and change back to using TimestampedTuple.
+ public static class TimestampExtractor implements com.google.common.base.Function<TempWrapper, Long>
+ {
+ @Override
+ public Long apply(@Nullable TempWrapper input)
+ {
+ return input.getTimestamp();
+ }
+ }
+
+ /**
+ * A temporary wrapper to wrap a KeyValPair and a timestamp together to represent a timestamped tuple, the reason
+ * for this is that we cannot resolve a type conflict when calling accumulate(). After the issue resolved, we can
+ * remove this class.
+ */
+ public static class TempWrapper
+ {
+ private KeyValPair<String, Long> value;
+ private Long timestamp;
+
+ public TempWrapper()
+ {
+
+ }
+
+ public TempWrapper(KeyValPair<String, Long> value, Long timestamp)
+ {
+ this.value = value;
+ this.timestamp = timestamp;
+ }
+
+ @Override
+ public String toString()
+ {
+ return this.value + " - " + this.timestamp;
+ }
+
+ public Long getTimestamp()
+ {
+ return timestamp;
+ }
+
+ public void setTimestamp(Long timestamp)
+ {
+ this.timestamp = timestamp;
+ }
+
+ public KeyValPair<String, Long> getValue()
+ {
+ return value;
+ }
+
+ public void setValue(KeyValPair<String, Long> value)
+ {
+ this.value = value;
+ }
+ }
+
+ /**
+ * Computes the longest session ending in each month, in this case we use 30 days to represent every month.
+ */
+ private static class TopPerMonth
+ extends CompositeStreamTransform<ApexStream<Tuple.WindowedTuple<KeyValPair<String, Long>>>, WindowedStream<Tuple.WindowedTuple<List<TempWrapper>>>>
+ {
+
+ @Override
+ public WindowedStream<Tuple.WindowedTuple<List<TempWrapper>>> compose(ApexStream<Tuple.WindowedTuple<KeyValPair<String, Long>>> inputStream)
+ {
+ TopN<TempWrapper> topN = new TopN<>();
+ topN.setN(10);
+ topN.setComparator(new Comp());
+
+ return inputStream
+
+ // Map the input WindowedTuple to a TempWrapper object.
+ .map(new Function.MapFunction<Tuple.WindowedTuple<KeyValPair<String, Long>>, TempWrapper>()
+ {
+ @Override
+ public TempWrapper f(Tuple.WindowedTuple<KeyValPair<String, Long>> input)
+ {
+ Window window = input.getWindows().iterator().next();
+ return new TempWrapper(input.getValue(), window.getBeginTimestamp());
+ }
+ }, name("TempWrapper"))
+
+ // Apply window and trigger option again, this time chuck the stream into fixed time windows.
+ .window(new WindowOption.TimeWindows(Duration.standardDays(30)), new TriggerOption().accumulatingFiredPanes().withEarlyFiringsAtEvery(Duration.standardSeconds(5)))
+
+ // Compute the top 10 user-sessions with most number of edits.
+ .accumulate(topN, name("TopN")).with("timestampExtractor", new TimestampExtractor());
+ }
+ }
+
+ /**
+ * A map function that combine the user and his/her edit session together to a string and use that string as a key
+ * with number of edits in that session as value to create a new key value pair to send to downstream.
+ */
+ static class SessionsToStringsDoFn implements Function.MapFunction<Tuple.WindowedTuple<KeyValPair<String, Long>>, Tuple.WindowedTuple<KeyValPair<String, Long>>>
+ {
+ @Override
+ public Tuple.WindowedTuple<KeyValPair<String, Long>> f(Tuple.WindowedTuple<KeyValPair<String, Long>> input)
+ {
+ Window window = input.getWindows().iterator().next();
+ return new Tuple.WindowedTuple<KeyValPair<String, Long>>(window, new KeyValPair<String, Long>(
+ input.getValue().getKey() + " : " + window.getBeginTimestamp() + " : " + window.getDurationMillis(),
+ input.getValue().getValue()));
+ }
+ }
+
+ /**
+ * A flatmap function that turns the result into readable format.
+ */
+ static class FormatOutputDoFn implements Function.FlatMapFunction<Tuple.WindowedTuple<List<TempWrapper>>, String>
+ {
+ @Override
+ public Iterable<String> f(Tuple.WindowedTuple<List<TempWrapper>> input)
+ {
+ ArrayList<String> result = new ArrayList<>();
+ for (TempWrapper item : input.getValue()) {
+ String session = item.getValue().getKey();
+ long count = item.getValue().getValue();
+ Window window = input.getWindows().iterator().next();
+ result.add(session + " + " + count + " : " + window.getBeginTimestamp());
+ }
+ return result;
+ }
+ }
+
+ /**
+ * A composite transform that compute the top wikipedia sessions.
+ */
+ public static class ComputeTopSessions extends CompositeStreamTransform<ApexStream<KeyValPair<String, Long>>, WindowedStream<Tuple.WindowedTuple<List<TempWrapper>>>>
+ {
+ @Override
+ public WindowedStream<Tuple.WindowedTuple<List<TempWrapper>>> compose(ApexStream<KeyValPair<String, Long>> inputStream)
+ {
+ return inputStream
+ .map(new ExtractUserAndTimestamp(), name("ExtractUserAndTimestamp"))
+ .addCompositeStreams(new ComputeSessions())
+ .map(new SessionsToStringsDoFn(), name("SessionsToStringsDoFn"))
+ .addCompositeStreams(new TopPerMonth());
+ }
+ }
+
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ SessionGen sg = new SessionGen();
+ Collector collector = new Collector();
+ StreamFactory.fromInput(sg, sg.output, name("sessionGen"))
+ .addCompositeStreams(new ComputeTopSessions())
+ .print(name("console"))
+ .endWith(collector, collector.input, name("collector")).populateDag(dag);
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/TrafficRoutes.java
----------------------------------------------------------------------
diff --git a/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/TrafficRoutes.java b/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/TrafficRoutes.java
new file mode 100644
index 0000000..e6a53d6
--- /dev/null
+++ b/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/TrafficRoutes.java
@@ -0,0 +1,523 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.malhar.stream.sample.complete;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Hashtable;
+import java.util.List;
+import java.util.Map;
+
+import javax.annotation.Nullable;
+
+import org.joda.time.Duration;
+
+import org.apache.apex.malhar.lib.window.TriggerOption;
+import org.apache.apex.malhar.lib.window.Tuple;
+import org.apache.apex.malhar.lib.window.WindowOption;
+import org.apache.apex.malhar.lib.window.accumulation.Group;
+import org.apache.apex.malhar.stream.api.ApexStream;
+import org.apache.apex.malhar.stream.api.CompositeStreamTransform;
+import org.apache.apex.malhar.stream.api.WindowedStream;
+import org.apache.apex.malhar.stream.api.function.Function;
+import org.apache.apex.malhar.stream.api.impl.StreamFactory;
+import org.apache.hadoop.conf.Configuration;
+
+import com.google.common.collect.Lists;
+
+import com.datatorrent.api.Context;
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.DefaultInputPort;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.InputOperator;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.common.util.BaseOperator;
+import com.datatorrent.lib.util.KeyValPair;
+
+import static org.apache.apex.malhar.stream.api.Option.Options.name;
+
+/**
+ * Beam's TrafficRoutes example.
+ *
+ * @since 3.5.0
+ */
+@ApplicationAnnotation(name = "TrafficRoutes")
+public class TrafficRoutes implements StreamingApplication
+{
+ static Map<String, String> sdStations = buildStationInfo();
+ static final int WINDOW_DURATION = 3; // Default sliding window duration in minutes
+ static final int WINDOW_SLIDE_EVERY = 1; // Default window 'slide every' setting in minutes
+
+ /**
+ * This class holds information about a station reading's average speed.
+ */
+ public static class StationSpeed implements Comparable<StationSpeed>
+ {
+ @Nullable
+ String stationId;
+ @Nullable
+ Double avgSpeed;
+ @Nullable
+ Long timestamp;
+
+ public StationSpeed() {}
+
+ public StationSpeed(String stationId, Double avgSpeed, Long timestamp)
+ {
+ this.stationId = stationId;
+ this.avgSpeed = avgSpeed;
+ this.timestamp = timestamp;
+ }
+
+ public void setAvgSpeed(@Nullable Double avgSpeed)
+ {
+ this.avgSpeed = avgSpeed;
+ }
+
+ public void setStationId(@Nullable String stationId)
+ {
+ this.stationId = stationId;
+ }
+
+ public void setTimestamp(@Nullable Long timestamp)
+ {
+ this.timestamp = timestamp;
+ }
+
+ @Nullable
+ public Long getTimestamp()
+ {
+ return timestamp;
+ }
+
+ public String getStationId()
+ {
+ return this.stationId;
+ }
+
+ public Double getAvgSpeed()
+ {
+ return this.avgSpeed;
+ }
+
+ @Override
+ public int compareTo(StationSpeed other)
+ {
+ return Long.compare(this.timestamp, other.timestamp);
+ }
+ }
+
+ /**
+ * This class holds information about a route's speed/slowdown.
+ */
+ static class RouteInfo
+ {
+ @Nullable
+ String route;
+ @Nullable
+ Double avgSpeed;
+ @Nullable
+ Boolean slowdownEvent;
+
+ public RouteInfo()
+ {
+
+ }
+
+ public RouteInfo(String route, Double avgSpeed, Boolean slowdownEvent)
+ {
+ this.route = route;
+ this.avgSpeed = avgSpeed;
+ this.slowdownEvent = slowdownEvent;
+ }
+
+ public String getRoute()
+ {
+ return this.route;
+ }
+
+ public Double getAvgSpeed()
+ {
+ return this.avgSpeed;
+ }
+
+ public Boolean getSlowdownEvent()
+ {
+ return this.slowdownEvent;
+ }
+ }
+
+ /**
+ * Extract the timestamp field from the input string, and wrap the input string in a {@link Tuple.TimestampedTuple}
+ * with the extracted timestamp.
+ */
+ static class ExtractTimestamps implements Function.MapFunction<String, Tuple.TimestampedTuple<String>>
+ {
+
+ @Override
+ public Tuple.TimestampedTuple<String> f(String input)
+ {
+ String[] items = input.split(",");
+ String timestamp = tryParseTimestamp(items);
+
+ return new Tuple.TimestampedTuple<>(Long.parseLong(timestamp), input);
+ }
+ }
+
+ /**
+ * Filter out readings for the stations along predefined 'routes', and output
+ * (station, speed info) keyed on route.
+ */
+ static class ExtractStationSpeedFn implements Function.FlatMapFunction<Tuple.TimestampedTuple<String>, KeyValPair<String, StationSpeed>>
+ {
+
+ @Override
+ public Iterable<KeyValPair<String, StationSpeed>> f(Tuple.TimestampedTuple<String> input)
+ {
+
+ ArrayList<KeyValPair<String, StationSpeed>> result = new ArrayList<>();
+ String[] items = input.getValue().split(",");
+ String stationType = tryParseStationType(items);
+ // For this analysis, use only 'main line' station types
+ if (stationType != null && stationType.equals("ML")) {
+ Double avgSpeed = tryParseAvgSpeed(items);
+ String stationId = tryParseStationId(items);
+ // For this simple example, filter out everything but some hardwired routes.
+ if (avgSpeed != null && stationId != null && sdStations.containsKey(stationId)) {
+ StationSpeed stationSpeed =
+ new StationSpeed(stationId, avgSpeed, input.getTimestamp());
+ // The tuple key is the 'route' name stored in the 'sdStations' hash.
+ KeyValPair<String, StationSpeed> outputValue = new KeyValPair<>(sdStations.get(stationId), stationSpeed);
+ result.add(outputValue);
+ }
+ }
+ return result;
+ }
+ }
+
+ /**
+ * For a given route, track average speed for the window. Calculate whether
+ * traffic is currently slowing down, via a predefined threshold. If a supermajority of
+ * speeds in this sliding window are less than the previous reading we call this a 'slowdown'.
+ * Note: these calculations are for example purposes only, and are unrealistic and oversimplified.
+ */
+ static class GatherStats
+ implements Function.FlatMapFunction<Tuple.WindowedTuple<KeyValPair<String, List<StationSpeed>>>, Tuple.TimestampedTuple<KeyValPair<String, RouteInfo>>>
+ {
+ @Override
+ public Iterable<Tuple.TimestampedTuple<KeyValPair<String, RouteInfo>>> f(Tuple.WindowedTuple<KeyValPair<String, List<StationSpeed>>> input)
+ {
+ ArrayList<Tuple.TimestampedTuple<KeyValPair<String, RouteInfo>>> result = new ArrayList<>();
+ String route = input.getValue().getKey();
+ double speedSum = 0.0;
+ int speedCount = 0;
+ int speedups = 0;
+ int slowdowns = 0;
+ List<StationSpeed> infoList = Lists.newArrayList(input.getValue().getValue());
+ // StationSpeeds sort by embedded timestamp.
+ Collections.sort(infoList);
+ Map<String, Double> prevSpeeds = new HashMap<>();
+ // For all stations in the route, sum (non-null) speeds. Keep a count of the non-null speeds.
+ for (StationSpeed item : infoList) {
+ Double speed = item.getAvgSpeed();
+ if (speed != null) {
+ speedSum += speed;
+ speedCount++;
+ Double lastSpeed = prevSpeeds.get(item.getStationId());
+ if (lastSpeed != null) {
+ if (lastSpeed < speed) {
+ speedups += 1;
+ } else {
+ slowdowns += 1;
+ }
+ }
+ prevSpeeds.put(item.getStationId(), speed);
+ }
+ }
+ if (speedCount == 0) {
+ // No average to compute.
+ return result;
+ }
+ double speedAvg = speedSum / speedCount;
+ boolean slowdownEvent = slowdowns >= 2 * speedups;
+ RouteInfo routeInfo = new RouteInfo(route, speedAvg, slowdownEvent);
+ result.add(new Tuple.TimestampedTuple<KeyValPair<String, RouteInfo>>(input.getTimestamp(), new KeyValPair<String, RouteInfo>(route, routeInfo)));
+ return result;
+ }
+ }
+
+ /**
+ * Output Pojo class for outputting result to JDBC.
+ */
+ static class OutputPojo
+ {
+ private Double avgSpeed;
+ private Boolean slowdownEvent;
+ private String key;
+ private Long timestamp;
+
+ public OutputPojo()
+ {
+ }
+
+ public OutputPojo(Double avgSpeed, Boolean slowdownEvent, String key, Long timestamp)
+ {
+ this.avgSpeed = avgSpeed;
+ this.slowdownEvent = slowdownEvent;
+ this.key = key;
+ this.timestamp = timestamp;
+ }
+
+ @Override
+ public String toString()
+ {
+ return key + " + " + avgSpeed + " + " + slowdownEvent + " + " + timestamp;
+ }
+
+ public void setTimestamp(Long timestamp)
+ {
+ this.timestamp = timestamp;
+ }
+
+ public Long getTimestamp()
+ {
+ return timestamp;
+ }
+
+ public void setAvgSpeed(Double avgSpeed)
+ {
+ this.avgSpeed = avgSpeed;
+ }
+
+ public Double getAvgSpeed()
+ {
+ return avgSpeed;
+ }
+
+ public void setKey(String key)
+ {
+ this.key = key;
+ }
+
+ public String getKey()
+ {
+ return key;
+ }
+
+ public void setSlowdownEvent(Boolean slowdownEvent)
+ {
+ this.slowdownEvent = slowdownEvent;
+ }
+
+ public Boolean getSlowdownEvent()
+ {
+ return slowdownEvent;
+ }
+
+ }
+
+ public static class Collector extends BaseOperator
+ {
+ private static Map<KeyValPair<Long, String>, KeyValPair<Double, Boolean>> result = new HashMap<>();
+
+ public static Map<KeyValPair<Long, String>, KeyValPair<Double, Boolean>> getResult()
+ {
+ return result;
+ }
+
+ public final transient DefaultInputPort<OutputPojo> input = new DefaultInputPort<OutputPojo>()
+ {
+ @Override
+ public void process(OutputPojo tuple)
+ {
+ result.put(new KeyValPair<Long, String>(tuple.getTimestamp(), tuple.getKey()), new KeyValPair<Double, Boolean>(tuple.getAvgSpeed(), tuple.getSlowdownEvent()));
+ }
+ };
+ }
+
+ /**
+ * Format the results of the slowdown calculations to a OutputPojo.
+ */
+ static class FormatStatsFn implements Function.MapFunction<Tuple.TimestampedTuple<KeyValPair<String, RouteInfo>>, OutputPojo>
+ {
+ @Override
+ public OutputPojo f(Tuple.TimestampedTuple<KeyValPair<String, RouteInfo>> input)
+ {
+ RouteInfo routeInfo = input.getValue().getValue();
+ OutputPojo row = new OutputPojo(routeInfo.getAvgSpeed(), routeInfo.getSlowdownEvent(), input.getValue().getKey(), input.getTimestamp());
+ return row;
+ }
+ }
+
+
+ /**
+ * This composite transformation extracts speed info from traffic station readings.
+ * It groups the readings by 'route' and analyzes traffic slowdown for that route.
+ * Lastly, it formats the results for JDBC.
+ */
+ static class TrackSpeed extends
+ CompositeStreamTransform<WindowedStream<KeyValPair<String, StationSpeed>>, WindowedStream<OutputPojo>>
+ {
+ @Override
+ public WindowedStream<OutputPojo> compose(WindowedStream<KeyValPair<String, StationSpeed>> inputStream)
+ {
+ // Apply a GroupByKey transform to collect a list of all station
+ // readings for a given route.
+ WindowedStream<Tuple.WindowedTuple<KeyValPair<String, List<StationSpeed>>>> timeGroup =
+ inputStream
+ .accumulateByKey(new Group<StationSpeed>(), new Function.ToKeyValue<KeyValPair<String, StationSpeed>, String, StationSpeed>()
+ {
+ @Override
+ public Tuple<KeyValPair<String, StationSpeed>> f(KeyValPair<String, StationSpeed> input)
+ {
+ return new Tuple.TimestampedTuple<>(input.getValue().getTimestamp(), input);
+ }
+ }, name("GroupByKey"));
+
+ // Analyze 'slowdown' over the route readings.
+ WindowedStream<Tuple.TimestampedTuple<KeyValPair<String, RouteInfo>>> stats = timeGroup
+ .flatMap(new GatherStats(), name("GatherStats"));
+
+ // Format the results for writing to JDBC table.
+ WindowedStream<OutputPojo> results = stats.map(new FormatStatsFn(), name("FormatStatsFn"));
+
+ return results;
+ }
+ }
+
+
+ private static Double tryParseAvgSpeed(String[] inputItems)
+ {
+ try {
+ return Double.parseDouble(tryParseString(inputItems, 3));
+ } catch (NumberFormatException e) {
+ return null;
+ } catch (NullPointerException e) {
+ return null;
+ }
+ }
+
+ private static String tryParseStationType(String[] inputItems)
+ {
+ return tryParseString(inputItems, 2);
+ }
+
+ private static String tryParseStationId(String[] inputItems)
+ {
+ return tryParseString(inputItems, 1);
+ }
+
+ private static String tryParseTimestamp(String[] inputItems)
+ {
+ return tryParseString(inputItems, 0);
+ }
+
+ private static String tryParseString(String[] inputItems, int index)
+ {
+ return inputItems.length >= index ? inputItems[index] : null;
+ }
+
+ /**
+ * Define some small hard-wired San Diego 'routes' to track based on sensor station ID.
+ */
+ private static Map<String, String> buildStationInfo()
+ {
+ Map<String, String> stations = new Hashtable<String, String>();
+ stations.put("1108413", "SDRoute1"); // from freeway 805 S
+ stations.put("1108699", "SDRoute2"); // from freeway 78 E
+ stations.put("1108702", "SDRoute2");
+ return stations;
+ }
+
+ /**
+ * A dummy generator to generate some traffic information.
+ */
+ public static class InfoGen extends BaseOperator implements InputOperator
+ {
+ public transient DefaultOutputPort<String> output = new DefaultOutputPort<>();
+
+ private String[] stationTypes = new String[]{"ML", "BL", "GL"};
+ private int[] stationIDs = new int[]{1108413, 1108699, 1108702};
+ private double ave = 55.0;
+ private long timestamp;
+ private static final Duration RAND_RANGE = Duration.standardMinutes(10);
+ private static int tupleCount = 0;
+
+ public static int getTupleCount()
+ {
+ return tupleCount;
+ }
+
+ @Override
+ public void setup(Context.OperatorContext context)
+ {
+ tupleCount = 0;
+ timestamp = System.currentTimeMillis();
+ }
+
+ @Override
+ public void emitTuples()
+ {
+ for (String stationType : stationTypes) {
+ for (int stationID : stationIDs) {
+ double speed = Math.random() * 20 + ave;
+ long time = (long)(Math.random() * RAND_RANGE.getMillis()) + timestamp;
+ try {
+ output.emit(time + "," + stationID + "," + stationType + "," + speed);
+ tupleCount++;
+
+ Thread.sleep(50);
+ } catch (Exception e) {
+ // Ignore it
+ }
+ }
+ }
+ }
+ }
+
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ InfoGen infoGen = new InfoGen();
+ Collector collector = new Collector();
+
+ // Create a stream from the input operator.
+ ApexStream<Tuple.TimestampedTuple<String>> stream = StreamFactory.fromInput(infoGen, infoGen.output, name("infoGen"))
+
+ // Extract the timestamp from the input and wrap it into a TimestampedTuple.
+ .map(new ExtractTimestamps(), name("ExtractTimestamps"));
+
+ stream
+ // Extract the average speed of a station.
+ .flatMap(new ExtractStationSpeedFn(), name("ExtractStationSpeedFn"))
+
+ // Apply window and trigger option.
+ .window(new WindowOption.SlidingTimeWindows(Duration.standardMinutes(WINDOW_DURATION), Duration.standardMinutes(WINDOW_SLIDE_EVERY)), new TriggerOption().withEarlyFiringsAtEvery(Duration.millis(5000)).accumulatingFiredPanes())
+
+ // Apply TrackSpeed composite transformation to compute the route information.
+ .addCompositeStreams(new TrackSpeed())
+
+ // print the result to console.
+ .print(name("console"))
+ .endWith(collector, collector.input, name("Collector"))
+ .populateDag(dag);
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/TwitterAutoComplete.java
----------------------------------------------------------------------
diff --git a/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/TwitterAutoComplete.java b/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/TwitterAutoComplete.java
new file mode 100644
index 0000000..6332c66
--- /dev/null
+++ b/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/TwitterAutoComplete.java
@@ -0,0 +1,254 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.malhar.stream.sample.complete;
+
+import java.nio.charset.Charset;
+import java.nio.charset.CharsetEncoder;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.joda.time.Duration;
+
+import org.apache.apex.malhar.lib.window.TriggerOption;
+import org.apache.apex.malhar.lib.window.Tuple;
+import org.apache.apex.malhar.lib.window.Window;
+import org.apache.apex.malhar.lib.window.WindowOption;
+
+import org.apache.apex.malhar.stream.api.ApexStream;
+import org.apache.apex.malhar.stream.api.CompositeStreamTransform;
+import org.apache.apex.malhar.stream.api.WindowedStream;
+import org.apache.apex.malhar.stream.api.function.Function;
+import org.apache.apex.malhar.stream.api.impl.StreamFactory;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.contrib.twitter.TwitterSampleInput;
+import com.datatorrent.lib.util.KeyValPair;
+
+import static org.apache.apex.malhar.stream.api.Option.Options.name;
+
+/**
+ * Auto Complete Hashtag Example with real time twitter input. In order to run this application, you need to create an app
+ * at https://apps.twitter.com, then generate your consumer and access keys and tokens, and enter those information
+ * accordingly in /resources/META-INF/properties.xml.
+ *
+ * The authentication requires following 4 information.
+ * Your application consumer key,
+ * Your application consumer secret,
+ * Your twitter access token, and
+ * Your twitter access token secret.
+ *
+ * @since 3.5.0
+ */
+@ApplicationAnnotation(name = "TwitterAutoComplete")
+public class TwitterAutoComplete implements StreamingApplication
+{
+ /**
+ * Check whether every character in a string is ASCII encoding.
+ */
+ public static class StringUtils
+ {
+ static CharsetEncoder encoder = Charset.forName("US-ASCII").newEncoder();
+
+ public static boolean isAscii(String v)
+ {
+ return encoder.canEncode(v);
+ }
+ }
+
+ /**
+ * FlapMap Function to extract all hashtags from a text form tweet.
+ */
+ private static class ExtractHashtags implements Function.FlatMapFunction<String, String>
+ {
+
+ @Override
+ public Iterable<String> f(String input)
+ {
+ List<String> result = new LinkedList<>();
+ Matcher m = Pattern.compile("#\\S+").matcher(input);
+ while (m.find()) {
+ result.add(m.group().substring(1));
+ }
+ return result;
+ }
+ }
+
+ /**
+ * Lower latency, but more expensive.
+ */
+ private static class ComputeTopFlat
+ extends CompositeStreamTransform<WindowedStream<CompletionCandidate>, WindowedStream<Tuple.WindowedTuple<KeyValPair<String, List<CompletionCandidate>>>>>
+ {
+ private final int candidatesPerPrefix;
+ private final int minPrefix;
+
+ public ComputeTopFlat(int candidatesPerPrefix, int minPrefix)
+ {
+ this.candidatesPerPrefix = candidatesPerPrefix;
+ this.minPrefix = minPrefix;
+ }
+
+ @Override
+ public WindowedStream<Tuple.WindowedTuple<KeyValPair<String, List<CompletionCandidate>>>> compose(
+ WindowedStream<CompletionCandidate> input)
+ {
+ TopNByKey topNByKey = new TopNByKey();
+ topNByKey.setN(candidatesPerPrefix);
+ return input
+ .<KeyValPair<String, CompletionCandidate>, WindowedStream<KeyValPair<String, CompletionCandidate>>>flatMap(new AllPrefixes(minPrefix, 3), name("Extract Prefixes"))
+ .accumulateByKey(topNByKey, new Function.ToKeyValue<KeyValPair<String, CompletionCandidate>, String, CompletionCandidate>()
+ {
+ @Override
+ public Tuple<KeyValPair<String, CompletionCandidate>> f(KeyValPair<String, CompletionCandidate> tuple)
+ {
+ // TODO: Should be removed after Auto-wrapping is supported.
+ return new Tuple.WindowedTuple<>(Window.GlobalWindow.INSTANCE, tuple);
+ }
+ }, name("TopNByKey"));
+ }
+ }
+
+ /**
+ * FlapMap Function to extract all prefixes of the hashtag in the input CompletionCandidate, and output
+ * KeyValPairs of the prefix and the CompletionCandidate
+ */
+ private static class AllPrefixes implements Function.FlatMapFunction<CompletionCandidate, KeyValPair<String, CompletionCandidate>>
+ {
+ private final int minPrefix;
+ private final int maxPrefix;
+
+ public AllPrefixes()
+ {
+ this(0, Integer.MAX_VALUE);
+ }
+
+ public AllPrefixes(int minPrefix)
+ {
+ this(minPrefix, Integer.MAX_VALUE);
+ }
+
+ public AllPrefixes(int minPrefix, int maxPrefix)
+ {
+ this.minPrefix = minPrefix;
+ this.maxPrefix = maxPrefix;
+ }
+
+ @Override
+ public Iterable<KeyValPair<String, CompletionCandidate>> f(CompletionCandidate input)
+ {
+ List<KeyValPair<String, CompletionCandidate>> result = new LinkedList<>();
+ String word = input.getValue();
+ for (int i = minPrefix; i <= Math.min(word.length(), maxPrefix); i++) {
+ result.add(new KeyValPair<>(input.getValue().substring(0, i).toLowerCase(), input));
+ }
+ return result;
+ }
+ }
+
+ /**
+ * A Composite stream transform that takes as input a list of tokens and returns
+ * the most common tokens per prefix.
+ */
+ public static class ComputeTopCompletions
+ extends CompositeStreamTransform<WindowedStream<String>, WindowedStream<Tuple.WindowedTuple<KeyValPair<String, List<CompletionCandidate>>>>>
+ {
+ private final int candidatesPerPrefix;
+ private final boolean recursive;
+
+ protected ComputeTopCompletions(int candidatesPerPrefix, boolean recursive)
+ {
+ this.candidatesPerPrefix = candidatesPerPrefix;
+ this.recursive = recursive;
+ }
+
+ public static ComputeTopCompletions top(int candidatesPerPrefix, boolean recursive)
+ {
+ return new ComputeTopCompletions(candidatesPerPrefix, recursive);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public WindowedStream<Tuple.WindowedTuple<KeyValPair<String, List<CompletionCandidate>>>> compose(WindowedStream<String> inputStream)
+ {
+
+ ApexStream<CompletionCandidate> candidates = inputStream
+ .countByKey(new Function.ToKeyValue<String, String, Long>()
+ {
+ @Override
+ public Tuple<KeyValPair<String, Long>> f(String input)
+ {
+ return new Tuple.PlainTuple<>(new KeyValPair<>(input, 1L));
+ }
+ }, name("Hashtag Count"))
+ .map(new Function.MapFunction<Tuple.WindowedTuple<KeyValPair<String,Long>>, CompletionCandidate>()
+ {
+ @Override
+ public CompletionCandidate f(Tuple.WindowedTuple<KeyValPair<String, Long>> input)
+ {
+ return new CompletionCandidate(input.getValue().getKey(), input.getValue().getValue());
+ }
+ }, name("KeyValPair to CompletionCandidate"));
+
+ return candidates.addCompositeStreams(new ComputeTopFlat(candidatesPerPrefix, 1));
+
+ }
+ }
+
+ /**
+ * FilterFunction to filter out tweets with non-acsii characters.
+ */
+ static class ASCIIFilter implements Function.FilterFunction<String>
+ {
+ @Override
+ public boolean f(String input)
+ {
+ return StringUtils.isAscii(input);
+ }
+ }
+
+ /**
+ * Populate the dag with High-Level API.
+ * @param dag
+ * @param conf
+ */
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ TwitterSampleInput input = new TwitterSampleInput();
+
+ WindowOption windowOption = new WindowOption.GlobalWindow();
+
+ ApexStream<String> tags = StreamFactory.fromInput(input, input.text, name("tweetSampler"))
+ .filter(new ASCIIFilter(), name("ACSII Filter"))
+ .flatMap(new ExtractHashtags(), name("Extract Hashtags"));
+
+ ApexStream<Tuple.WindowedTuple<KeyValPair<String, List<CompletionCandidate>>>> s =
+ tags.window(windowOption, new TriggerOption().accumulatingFiredPanes().withEarlyFiringsAtEvery(Duration.standardSeconds(10)))
+ .addCompositeStreams(ComputeTopCompletions.top(10, true)).print();
+
+ s.populateDag(dag);
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/cookbook/CombinePerKeyExamples.java
----------------------------------------------------------------------
diff --git a/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/cookbook/CombinePerKeyExamples.java b/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/cookbook/CombinePerKeyExamples.java
new file mode 100644
index 0000000..bfdb268
--- /dev/null
+++ b/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/cookbook/CombinePerKeyExamples.java
@@ -0,0 +1,285 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.malhar.stream.sample.cookbook;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.apex.malhar.lib.window.TriggerOption;
+import org.apache.apex.malhar.lib.window.Tuple;
+import org.apache.apex.malhar.lib.window.WindowOption;
+import org.apache.apex.malhar.lib.window.accumulation.ReduceFn;
+import org.apache.apex.malhar.stream.api.ApexStream;
+import org.apache.apex.malhar.stream.api.CompositeStreamTransform;
+import org.apache.apex.malhar.stream.api.WindowedStream;
+import org.apache.apex.malhar.stream.api.function.Function;
+import org.apache.apex.malhar.stream.api.impl.StreamFactory;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.Context;
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.DefaultInputPort;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.InputOperator;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.common.util.BaseOperator;
+import com.datatorrent.lib.util.KeyValPair;
+
+import static org.apache.apex.malhar.stream.api.Option.Options.name;
+
+/**
+ * An example that reads the public 'Shakespeare' data, and for each word in
+ * the dataset that is over a given length, generates a string containing the
+ * list of play names in which that word appears
+ *
+ * <p>Concepts: the combine transform, which lets you combine the values in a
+ * key-grouped Collection
+ *
+ *
+ * @since 3.5.0
+ */
+@ApplicationAnnotation(name = "CombinePerKeyExamples")
+public class CombinePerKeyExamples implements StreamingApplication
+{
+ // Use the shakespeare public BigQuery sample
+ private static final String SHAKESPEARE_TABLE = "publicdata:samples.shakespeare";
+ // We'll track words >= this word length across all plays in the table.
+ private static final int MIN_WORD_LENGTH = 0;
+
+ /**
+ * Examines each row in the input table. If the word is greater than or equal to MIN_WORD_LENGTH,
+ * outputs word, play_name.
+ */
+ static class ExtractLargeWordsFn implements Function.MapFunction<SampleBean, KeyValPair<String, String>>
+ {
+
+ @Override
+ public KeyValPair<String, String> f(SampleBean input)
+ {
+ String playName = input.getCorpus();
+ String word = input.getWord();
+ if (word.length() >= MIN_WORD_LENGTH) {
+ return new KeyValPair<>(word, playName);
+ } else {
+ return null;
+ }
+ }
+ }
+
+
+ /**
+ * Prepares the output data which is in same bean
+ */
+ static class FormatShakespeareOutputFn implements Function.MapFunction<Tuple.WindowedTuple<KeyValPair<String, String>>, SampleBean>
+ {
+ @Override
+ public SampleBean f(Tuple.WindowedTuple<KeyValPair<String, String>> input)
+ {
+ return new SampleBean(input.getValue().getKey(), input.getValue().getValue());
+ }
+ }
+
+ /**
+ * A reduce function to concat two strings together.
+ */
+ public static class Concat extends ReduceFn<String>
+ {
+ @Override
+ public String reduce(String input1, String input2)
+ {
+ return input1 + ", " + input2;
+ }
+ }
+
+ /**
+ * Reads the public 'Shakespeare' data, and for each word in the dataset
+ * over a given length, generates a string containing the list of play names
+ * in which that word appears.
+ */
+ private static class PlaysForWord extends CompositeStreamTransform<ApexStream<SampleBean>, WindowedStream<SampleBean>>
+ {
+
+ @Override
+ public WindowedStream<SampleBean> compose(ApexStream<SampleBean> inputStream)
+ {
+ return inputStream
+ // Extract words from the input SampleBeam stream.
+ .map(new ExtractLargeWordsFn(), name("ExtractLargeWordsFn"))
+
+ // Apply window and trigger option to the streams.
+ .window(new WindowOption.GlobalWindow(), new TriggerOption().accumulatingFiredPanes().withEarlyFiringsAtEvery(1))
+
+ // Apply reduceByKey transformation to concat the names of all the plays that a word has appeared in together.
+ .reduceByKey(new Concat(), new Function.ToKeyValue<KeyValPair<String,String>, String, String>()
+ {
+ @Override
+ public Tuple<KeyValPair<String, String>> f(KeyValPair<String, String> input)
+ {
+ return new Tuple.PlainTuple<KeyValPair<String, String>>(input);
+ }
+ }, name("Concat"))
+
+ // Format the output back to a SampleBeam object.
+ .map(new FormatShakespeareOutputFn(), name("FormatShakespeareOutputFn"));
+ }
+ }
+
+
+ /**
+ * A Java Beam class that contains information about a word appears in a corpus written by Shakespeare.
+ */
+ public static class SampleBean
+ {
+
+ public SampleBean()
+ {
+
+ }
+
+ public SampleBean(String word, String corpus)
+ {
+ this.word = word;
+ this.corpus = corpus;
+ }
+
+ @Override
+ public String toString()
+ {
+ return this.word + " : " + this.corpus;
+ }
+
+ private String word;
+
+ private String corpus;
+
+ public void setWord(String word)
+ {
+ this.word = word;
+ }
+
+ public String getWord()
+ {
+ return word;
+ }
+
+ public void setCorpus(String corpus)
+ {
+ this.corpus = corpus;
+ }
+
+ public String getCorpus()
+ {
+ return corpus;
+ }
+ }
+
+ /**
+ * A dummy info generator to generate {@link SampleBean} objects to mimic reading from real 'Shakespeare'
+ * data.
+ */
+ public static class SampleInput extends BaseOperator implements InputOperator
+ {
+
+ public final transient DefaultOutputPort<SampleBean> beanOutput = new DefaultOutputPort();
+ private String[] words = new String[]{"A", "B", "C", "D", "E", "F", "G"};
+ private String[] corpuses = new String[]{"1", "2", "3", "4", "5", "6", "7", "8"};
+ private static int i;
+
+ @Override
+ public void setup(Context.OperatorContext context)
+ {
+ super.setup(context);
+ i = 0;
+ }
+
+ @Override
+ public void emitTuples()
+ {
+ while (i < 1) {
+ for (String word : words) {
+ for (String corpus : corpuses) {
+ try {
+ Thread.sleep(50);
+ beanOutput.emit(new SampleBean(word, corpus));
+ } catch (Exception e) {
+ // Ignore it
+ }
+ }
+ }
+ i++;
+ }
+
+ }
+ }
+
+ public static class Collector extends BaseOperator
+ {
+ private static List<SampleBean> result;
+ private static boolean done = false;
+
+ public static List<SampleBean> getResult()
+ {
+ return result;
+ }
+
+ public static boolean isDone()
+ {
+ return done;
+ }
+
+ @Override
+ public void setup(Context.OperatorContext context)
+ {
+ result = new ArrayList<>();
+ done = false;
+ }
+
+ public final transient DefaultInputPort<SampleBean> input = new DefaultInputPort<SampleBean>()
+ {
+ @Override
+ public void process(SampleBean tuple)
+ {
+ if (tuple.getWord().equals("F")) {
+ done = true;
+ }
+ result.add(tuple);
+ }
+ };
+ }
+
+ /**
+ * Populate dag using High-Level API.
+ * @param dag
+ * @param conf
+ */
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ SampleInput input = new SampleInput();
+ Collector collector = new Collector();
+ StreamFactory.fromInput(input, input.beanOutput, name("input"))
+ .addCompositeStreams(new PlaysForWord())
+ .print(name("console"))
+ .endWith(collector, collector.input, name("Collector"))
+ .populateDag(dag);
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/cookbook/DeDupExample.java
----------------------------------------------------------------------
diff --git a/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/cookbook/DeDupExample.java b/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/cookbook/DeDupExample.java
new file mode 100644
index 0000000..4df5fe7
--- /dev/null
+++ b/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/cookbook/DeDupExample.java
@@ -0,0 +1,127 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.malhar.stream.sample.cookbook;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.joda.time.Duration;
+
+import org.apache.apex.malhar.lib.window.TriggerOption;
+import org.apache.apex.malhar.lib.window.Tuple;
+import org.apache.apex.malhar.lib.window.WindowOption;
+import org.apache.apex.malhar.lib.window.accumulation.RemoveDuplicates;
+import org.apache.apex.malhar.stream.api.ApexStream;
+import org.apache.apex.malhar.stream.api.function.Function;
+import org.apache.apex.malhar.stream.api.impl.StreamFactory;
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.Context;
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.DefaultInputPort;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.common.util.BaseOperator;
+
+import static org.apache.apex.malhar.stream.api.Option.Options.name;
+
+/**
+ * Beam DeDupExample.
+ *
+ * @since 3.5.0
+ */
+@ApplicationAnnotation(name = "DeDupExample")
+public class DeDupExample implements StreamingApplication
+{
+
+ public static class Collector extends BaseOperator
+ {
+ private static Tuple.WindowedTuple<List<String>> result;
+ private static boolean done = false;
+
+ public static Tuple.WindowedTuple<List<String>> getResult()
+ {
+ return result;
+ }
+
+ public static boolean isDone()
+ {
+ return done;
+ }
+
+ @Override
+ public void setup(Context.OperatorContext context)
+ {
+ super.setup(context);
+ result = new Tuple.WindowedTuple<>();
+ done = false;
+ }
+
+ public transient DefaultInputPort<Tuple.WindowedTuple<List<String>>> input = new DefaultInputPort<Tuple.WindowedTuple<List<String>>>()
+ {
+ @Override
+ public void process(Tuple.WindowedTuple<List<String>> tuple)
+ {
+ result = tuple;
+ if (result.getValue().contains("bye")) {
+ done = true;
+ }
+ }
+ };
+ }
+
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ Collector collector = new Collector();
+
+ // Create a stream that reads from files in a local folder and output lines one by one to downstream.
+ ApexStream<String> stream = StreamFactory.fromFolder("./src/test/resources/wordcount", name("textInput"))
+
+ // Extract all the words from the input line of text.
+ .flatMap(new Function.FlatMapFunction<String, String>()
+ {
+ @Override
+ public Iterable<String> f(String input)
+ {
+ return Arrays.asList(input.split("[\\p{Punct}\\s]+"));
+ }
+ }, name("ExtractWords"))
+
+ // Change the words to lower case, also shutdown the app when the word "bye" is detected.
+ .map(new Function.MapFunction<String, String>()
+ {
+ @Override
+ public String f(String input)
+ {
+ return input.toLowerCase();
+ }
+ }, name("ToLowerCase"));
+
+ // Apply window and trigger option.
+ stream.window(new WindowOption.GlobalWindow(),
+ new TriggerOption().accumulatingFiredPanes().withEarlyFiringsAtEvery(Duration.standardSeconds(1)))
+
+ // Remove the duplicate words and print out the result.
+ .accumulate(new RemoveDuplicates<String>(), name("RemoveDuplicates"))
+ .print(name("console"))
+ .endWith(collector, collector.input)
+ .populateDag(dag);
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/cookbook/InputPojo.java
----------------------------------------------------------------------
diff --git a/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/cookbook/InputPojo.java b/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/cookbook/InputPojo.java
new file mode 100644
index 0000000..834964c
--- /dev/null
+++ b/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/cookbook/InputPojo.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.malhar.stream.sample.cookbook;
+
+/**
+ * Tuple class for JDBC input of {@link MaxPerKeyExamples}.
+ *
+ * @since 3.5.0
+ */
+public class InputPojo extends Object
+{
+ private int month;
+ private int day;
+ private int year;
+ private double meanTemp;
+
+ @Override
+ public String toString()
+ {
+ return "PojoEvent [month=" + getMonth() + ", day=" + getDay() + ", year=" + getYear() + ", meanTemp=" + getMeanTemp() + "]";
+ }
+
+ public void setMonth(int month)
+ {
+ this.month = month;
+ }
+
+ public int getMonth()
+ {
+ return this.month;
+ }
+
+ public void setDay(int day)
+ {
+ this.day = day;
+ }
+
+ public int getDay()
+ {
+ return day;
+ }
+
+ public void setYear(int year)
+ {
+ this.year = year;
+ }
+
+ public int getYear()
+ {
+ return year;
+ }
+
+ public void setMeanTemp(double meanTemp)
+ {
+ this.meanTemp = meanTemp;
+ }
+
+ public double getMeanTemp()
+ {
+ return meanTemp;
+ }
+}
[29/30] apex-malhar git commit: Renamed demos to examples. Packages
and artifactid names are changed as suggested.
Posted by th...@apache.org.
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/distributedistinct/src/test/java/com/datatorrent/demos/distributeddistinct/StatefulUniqueCountTest.java
----------------------------------------------------------------------
diff --git a/demos/distributedistinct/src/test/java/com/datatorrent/demos/distributeddistinct/StatefulUniqueCountTest.java b/demos/distributedistinct/src/test/java/com/datatorrent/demos/distributeddistinct/StatefulUniqueCountTest.java
deleted file mode 100644
index a1ac603..0000000
--- a/demos/distributedistinct/src/test/java/com/datatorrent/demos/distributeddistinct/StatefulUniqueCountTest.java
+++ /dev/null
@@ -1,248 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.distributeddistinct;
-
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Properties;
-
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.Context.OperatorContext;
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.DefaultInputPort;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.api.InputOperator;
-import com.datatorrent.api.LocalMode;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.common.util.BaseOperator;
-
-import com.datatorrent.lib.algo.UniqueValueCount;
-import com.datatorrent.lib.util.KeyValPair;
-
-public class StatefulUniqueCountTest
-{
-
- public static final String INMEM_DB_URL = "jdbc:hsqldb:mem:test;sql.syntax_mys=true";
- public static final String INMEM_DB_DRIVER = "org.hsqldb.jdbc.JDBCDriver";
- public static final String TABLE_NAME = "Test_Lookup_Cache";
-
- static class KeyGen implements InputOperator
- {
-
- public transient DefaultOutputPort<KeyValPair<Integer, Object>> output = new DefaultOutputPort<KeyValPair<Integer, Object>>();
-
- @Override
- public void beginWindow(long windowId)
- {
- }
-
- public void emitKeyVals(int key, int start, int end, int increment)
- {
- for (int i = start; i <= end; i += increment) {
- output.emit(new KeyValPair<Integer, Object>(key, i));
- }
- }
-
- @Override
- public void endWindow()
- {
- }
-
- @Override
- public void setup(OperatorContext context)
- {
-
- }
-
- @Override
- public void teardown()
- {
-
- }
-
- @Override
- public void emitTuples()
- {
- emitKeyVals(1, 1, 10, 1);
- emitKeyVals(2, 3, 15, 3);
- emitKeyVals(3, 2, 20, 2);
- emitKeyVals(1, 5, 15, 1);
- emitKeyVals(2, 11, 20, 1);
- emitKeyVals(3, 11, 20, 1);
- }
- }
-
- static class VerifyTable extends BaseOperator
- {
-
- private static final String INMEM_DB_URL = "jdbc:hsqldb:mem:test;sql.syntax_mys=true";
- private static final String INMEM_DB_DRIVER = "org.hsqldb.jdbc.JDBCDriver";
- protected static final String TABLE_NAME = "Test_Lookup_Cache";
-
- public final transient DefaultInputPort<Object> input = new DefaultInputPort<Object>()
- {
- @Override
- public void process(Object tuple)
- {
- }
- };
-
- @Override
- public void beginWindow(long windowId)
- {
- }
-
- @Override
- public void endWindow()
- {
- try {
- Class.forName(INMEM_DB_DRIVER).newInstance();
- Connection con = DriverManager.getConnection(INMEM_DB_URL, new Properties());
- Statement stmt = con.createStatement();
- ResultSet resultSet = stmt.executeQuery("SELECT col2 FROM " + TABLE_NAME + " WHERE col1 = 1");
- ArrayList<Integer> answersOne = new ArrayList<Integer>();
- for (int i = 1; i < 16; i++) {
- answersOne.add(i);
- }
- Assert.assertEquals(answersOne, processResult(resultSet));
-
- resultSet = stmt.executeQuery("SELECT col2 FROM " + TABLE_NAME + " WHERE col1 = 2");
- ArrayList<Integer> answersTwo = new ArrayList<Integer>();
- answersTwo.add(3);
- answersTwo.add(6);
- answersTwo.add(9);
- for (int i = 11; i < 21; i++) {
- answersTwo.add(i);
- }
- Assert.assertEquals(answersTwo, processResult(resultSet));
-
- resultSet = stmt.executeQuery("SELECT col2 FROM " + TABLE_NAME + " WHERE col1 = 3");
- ArrayList<Integer> answersThree = new ArrayList<Integer>();
- answersThree.add(2);
- answersThree.add(4);
- answersThree.add(6);
- answersThree.add(8);
- answersThree.add(10);
- for (int i = 11; i < 21; i++) {
- answersThree.add(i);
- }
- Assert.assertEquals(answersThree, processResult(resultSet));
- } catch (Throwable e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public void setup(OperatorContext context)
- {
- }
-
- @Override
- public void teardown()
- {
-
- }
-
- public static ArrayList<Integer> processResult(ResultSet resultSet)
- {
- ArrayList<Integer> tempList = new ArrayList<Integer>();
- try {
- while (resultSet.next()) {
- tempList.add(resultSet.getInt(1));
- }
- } catch (SQLException e) {
- throw new RuntimeException(e);
- }
- Collections.sort(tempList);
- return tempList;
- }
- }
-
- public class Application implements StreamingApplication
- {
- @SuppressWarnings("unchecked")
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- KeyGen keyGen = dag.addOperator("KeyGenerator", new KeyGen());
- UniqueValueCount<Integer> valCount = dag.addOperator("ValueCounter", new UniqueValueCount<Integer>());
- IntegerUniqueValueCountAppender uniqueUnifier = dag.addOperator("Unique", new IntegerUniqueValueCountAppender());
- VerifyTable verifyTable = dag.addOperator("VerifyTable", new VerifyTable());
-
- @SuppressWarnings("rawtypes")
- DefaultOutputPort valOut = valCount.output;
- @SuppressWarnings("rawtypes")
- DefaultOutputPort uniqueOut = uniqueUnifier.output;
- dag.addStream("DataIn", keyGen.output, valCount.input);
- dag.addStream("UnifyWindows", valOut, uniqueUnifier.input);
- dag.addStream("ResultsOut", uniqueOut, verifyTable.input);
- }
- }
-
- @BeforeClass
- public static void setup()
- {
- try {
- Class.forName(INMEM_DB_DRIVER).newInstance();
- Connection con = DriverManager.getConnection(INMEM_DB_URL, new Properties());
- Statement stmt = con.createStatement();
- stmt.execute("CREATE TABLE IF NOT EXISTS " + TABLE_NAME + " (col1 INTEGER, col2 INTEGER, col3 BIGINT)");
- } catch (InstantiationException e) {
- throw new RuntimeException(e);
- } catch (IllegalAccessException e) {
- throw new RuntimeException(e);
- } catch (ClassNotFoundException e) {
- throw new RuntimeException(e);
- } catch (SQLException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Test
- public void testApplication() throws Exception
- {
- LocalMode lma = LocalMode.newInstance();
- Configuration conf = new Configuration(false);
- conf.set("dt.operator.Unique.prop.tableName", "Test_Lookup_Cache");
- conf.set("dt.operator.Unique.prop.store.dbUrl", "jdbc:hsqldb:mem:test;sql.syntax_mys=true");
- conf.set("dt.operator.Unique.prop.store.dbDriver", "org.hsqldb.jdbcDriver");
-
- lma.prepareDAG(new Application(), conf);
- lma.cloneDAG();
- LocalMode.Controller lc = lma.getController();
- lc.setHeartbeatMonitoringEnabled(false);
- lc.runAsync();
-
- long now = System.currentTimeMillis();
- while (System.currentTimeMillis() - now < 15000) {
- Thread.sleep(1000);
- }
- lc.shutdown();
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/distributedistinct/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/demos/distributedistinct/src/test/resources/log4j.properties b/demos/distributedistinct/src/test/resources/log4j.properties
deleted file mode 100644
index cf0d19e..0000000
--- a/demos/distributedistinct/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,43 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-log4j.rootLogger=DEBUG,CONSOLE
-
-log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
-log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
-log4j.appender.CONSOLE.threshold=${test.log.console.threshold}
-test.log.console.threshold=DEBUG
-
-log4j.appender.RFA=org.apache.log4j.RollingFileAppender
-log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
-log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
-log4j.appender.RFA.File=/tmp/app.log
-
-# to enable, add SYSLOG to rootLogger
-log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender
-log4j.appender.SYSLOG.syslogHost=127.0.0.1
-log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout
-log4j.appender.SYSLOG.layout.conversionPattern=${dt.cid} %-5p [%t] %c{2} %x - %m%n
-log4j.appender.SYSLOG.Facility=LOCAL1
-
-log4j.logger.org=info
-#log4j.logger.org.apache.commons.beanutils=warn
-log4j.logger.com.datatorrent=debug
-log4j.logger.org.apache.apex=debug
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/echoserver/pom.xml
----------------------------------------------------------------------
diff --git a/demos/echoserver/pom.xml b/demos/echoserver/pom.xml
deleted file mode 100644
index 603c678..0000000
--- a/demos/echoserver/pom.xml
+++ /dev/null
@@ -1,38 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <artifactId>malhar-demos</artifactId>
- <groupId>org.apache.apex</groupId>
- <version>3.7.0-SNAPSHOT</version>
- </parent>
-
- <groupId>org.apache.apex</groupId>
- <artifactId>echoserver</artifactId>
- <packaging>jar</packaging>
-
- <!-- change these to the appropriate values -->
- <name>Apache Apex Malhar EchoServer Demo</name>
- <description>A demo server that echos data sent by a network client back to it</description>
-</project>
-
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/echoserver/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/demos/echoserver/src/assemble/appPackage.xml b/demos/echoserver/src/assemble/appPackage.xml
deleted file mode 100644
index 4138cf2..0000000
--- a/demos/echoserver/src/assemble/appPackage.xml
+++ /dev/null
@@ -1,59 +0,0 @@
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
- <id>appPackage</id>
- <formats>
- <format>jar</format>
- </formats>
- <includeBaseDirectory>false</includeBaseDirectory>
- <fileSets>
- <fileSet>
- <directory>${basedir}/target/</directory>
- <outputDirectory>/app</outputDirectory>
- <includes>
- <include>${project.artifactId}-${project.version}.jar</include>
- </includes>
- </fileSet>
- <fileSet>
- <directory>${basedir}/target/deps</directory>
- <outputDirectory>/lib</outputDirectory>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/site/conf</directory>
- <outputDirectory>/conf</outputDirectory>
- <includes>
- <include>*.xml</include>
- </includes>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/main/resources/META-INF</directory>
- <outputDirectory>/META-INF</outputDirectory>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/main/resources/app</directory>
- <outputDirectory>/app</outputDirectory>
- </fileSet>
- </fileSets>
-
-</assembly>
-
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/echoserver/src/main/java/com/datatorrent/demos/echoserver/Application.java
----------------------------------------------------------------------
diff --git a/demos/echoserver/src/main/java/com/datatorrent/demos/echoserver/Application.java b/demos/echoserver/src/main/java/com/datatorrent/demos/echoserver/Application.java
deleted file mode 100644
index 90a3fd2..0000000
--- a/demos/echoserver/src/main/java/com/datatorrent/demos/echoserver/Application.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.echoserver;
-
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-
-/**
- * @since 2.1.0
- */
-@ApplicationAnnotation(name = "EchoServer")
-public class Application implements StreamingApplication
-{
-
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- MessageReceiver receiver = dag.addOperator("Message Receiver", MessageReceiver.class);
- MessageResponder responder = dag.addOperator("Message Responder", MessageResponder.class);
- // Locality has to be container so that the operators use the same socket
- dag.addStream("messages", receiver.messageOutput, responder.messageInput).setLocality(DAG.Locality.CONTAINER_LOCAL);
-
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/echoserver/src/main/java/com/datatorrent/demos/echoserver/Message.java
----------------------------------------------------------------------
diff --git a/demos/echoserver/src/main/java/com/datatorrent/demos/echoserver/Message.java b/demos/echoserver/src/main/java/com/datatorrent/demos/echoserver/Message.java
deleted file mode 100644
index 94ce1b7..0000000
--- a/demos/echoserver/src/main/java/com/datatorrent/demos/echoserver/Message.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.echoserver;
-
-import java.net.SocketAddress;
-
-/**
- * @since 2.1.0
- */
-public class Message
-{
- public String message;
- public SocketAddress socketAddress;
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/echoserver/src/main/java/com/datatorrent/demos/echoserver/MessageReceiver.java
----------------------------------------------------------------------
diff --git a/demos/echoserver/src/main/java/com/datatorrent/demos/echoserver/MessageReceiver.java b/demos/echoserver/src/main/java/com/datatorrent/demos/echoserver/MessageReceiver.java
deleted file mode 100644
index 6dce73f..0000000
--- a/demos/echoserver/src/main/java/com/datatorrent/demos/echoserver/MessageReceiver.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.echoserver;
-
-import java.io.IOException;
-import java.net.SocketAddress;
-import java.nio.ByteBuffer;
-import java.nio.channels.DatagramChannel;
-import java.nio.channels.SelectionKey;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.datatorrent.api.Context;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.api.InputOperator;
-
-/**
- * @since 2.1.0
- */
-public class MessageReceiver implements InputOperator, NetworkManager.ChannelListener<DatagramChannel>
-{
- private static final Logger logger = LoggerFactory.getLogger(MessageReceiver.class);
-
- private transient NetworkManager.ChannelAction<DatagramChannel> action;
-
- //Need the sender info, using a packet for now instead of the buffer
- private transient ByteBuffer buffer;
- //private transient DatagramPacket packet;
-
- private int port = 9000;
- private int maxMesgSize = 512;
- private int inactiveWait = 10;
- private boolean readReady = false;
-
- @Override
- public void emitTuples()
- {
- boolean emitData = false;
- if (readReady) {
- //DatagramSocket socket = action.channelConfiguration.socket;
- try {
- //socket.receive(packet);
- DatagramChannel channel = action.channelConfiguration.channel;
- SocketAddress address = channel.receive(buffer);
- if (address != null) {
- /*
- StringBuilder sb = new StringBuilder();
- buffer.rewind();
- while (buffer.hasRemaining()) {
- sb.append(buffer.getChar());
- }
- String mesg = sb.toString();
- */
- buffer.flip();
- String mesg = new String(buffer.array(), 0, buffer.limit());
- logger.info("Message {}", mesg);
- Message message = new Message();
- message.message = mesg;
- message.socketAddress = address;
- messageOutput.emit(message);
- emitData = true;
- buffer.clear();
- }
- //String mesg = new String(packet.getData(), packet.getOffset(), packet.getLength());
- } catch (IOException e) {
- throw new RuntimeException("Error reading from channel", e);
- }
- // Even if we miss a readReady because of not locking we will get it again immediately
- readReady = false;
- }
- if (!emitData) {
- synchronized (buffer) {
- try {
- if (!readReady) {
- buffer.wait(inactiveWait);
- }
- } catch (InterruptedException e) {
- throw new RuntimeException(e);
- }
- }
- }
- }
-
- @Override
- public void beginWindow(long l)
- {
-
- }
-
- @Override
- public void endWindow()
- {
-
- }
-
- public final transient DefaultOutputPort<Message> messageOutput = new DefaultOutputPort<Message>();
-
- @Override
- public void setup(Context.OperatorContext context)
- {
- try {
- //byte[] mesgData = new byte[maxMesgSize];
- //packet = new DatagramPacket(mesgData, maxMesgSize);
- buffer = ByteBuffer.allocate(maxMesgSize);
- action = NetworkManager.getInstance().registerAction(port, NetworkManager.ConnectionType.UDP, this, SelectionKey.OP_READ);
- } catch (IOException e) {
- throw new RuntimeException("Error initializing receiver", e);
- }
- }
-
- @Override
- public void teardown()
- {
- try {
- NetworkManager.getInstance().unregisterAction(action);
- } catch (Exception e) {
- throw new RuntimeException("Error shutting down receiver", e);
- }
- }
-
- @Override
- public void ready(NetworkManager.ChannelAction<DatagramChannel> action, int readyOps)
- {
- synchronized (buffer) {
- readReady = true;
- buffer.notify();
- }
- }
-
- public int getPort()
- {
- return port;
- }
-
- public void setPort(int port)
- {
- this.port = port;
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/echoserver/src/main/java/com/datatorrent/demos/echoserver/MessageResponder.java
----------------------------------------------------------------------
diff --git a/demos/echoserver/src/main/java/com/datatorrent/demos/echoserver/MessageResponder.java b/demos/echoserver/src/main/java/com/datatorrent/demos/echoserver/MessageResponder.java
deleted file mode 100644
index ce7a1bc..0000000
--- a/demos/echoserver/src/main/java/com/datatorrent/demos/echoserver/MessageResponder.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.echoserver;
-
-import java.io.IOException;
-import java.net.SocketAddress;
-import java.nio.ByteBuffer;
-import java.nio.channels.DatagramChannel;
-
-import com.datatorrent.api.Context;
-import com.datatorrent.api.DefaultInputPort;
-import com.datatorrent.common.util.BaseOperator;
-
-/**
- * @since 2.1.0
- */
-public class MessageResponder extends BaseOperator
-{
-
- private String responseHeader = "Response: ";
-
- private int port = 9000;
- private int maxMesgSize = 512;
- private transient NetworkManager.ChannelAction<DatagramChannel> action;
- private transient ByteBuffer buffer;
-
- public final transient DefaultInputPort<Message> messageInput = new DefaultInputPort<Message>()
- {
- @Override
- public void process(Message message)
- {
- String sendMesg = responseHeader + message.message;
- SocketAddress address = message.socketAddress;
- buffer.put(sendMesg.getBytes());
- buffer.flip();
- try {
- action.channelConfiguration.channel.send(buffer, address);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- buffer.clear();
- }
- };
-
- @Override
- public void setup(Context.OperatorContext context)
- {
- try {
- buffer = ByteBuffer.allocate(maxMesgSize);
- action = NetworkManager.getInstance().registerAction(port, NetworkManager.ConnectionType.UDP, null, 0);
- } catch (IOException e) {
- throw new RuntimeException("Error initializer responder", e);
- }
- }
-
- @Override
- public void teardown()
- {
- try {
- NetworkManager.getInstance().unregisterAction(action);
- } catch (Exception e) {
- throw new RuntimeException("Error shutting down responder", e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/echoserver/src/main/java/com/datatorrent/demos/echoserver/NetworkManager.java
----------------------------------------------------------------------
diff --git a/demos/echoserver/src/main/java/com/datatorrent/demos/echoserver/NetworkManager.java b/demos/echoserver/src/main/java/com/datatorrent/demos/echoserver/NetworkManager.java
deleted file mode 100644
index 056068f..0000000
--- a/demos/echoserver/src/main/java/com/datatorrent/demos/echoserver/NetworkManager.java
+++ /dev/null
@@ -1,249 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.echoserver;
-
-import java.io.IOException;
-import java.net.DatagramSocket;
-import java.net.InetSocketAddress;
-import java.net.Socket;
-import java.net.SocketAddress;
-import java.nio.channels.DatagramChannel;
-import java.nio.channels.SelectableChannel;
-import java.nio.channels.SelectionKey;
-import java.nio.channels.Selector;
-import java.nio.channels.SocketChannel;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentLinkedQueue;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @since 2.1.0
- */
-public class NetworkManager implements Runnable
-{
- private static final Logger logger = LoggerFactory.getLogger(NetworkManager.class);
-
- public static enum ConnectionType
- {
- TCP,
- UDP
- }
-
- private static NetworkManager _instance;
- private Selector selector;
-
- private volatile boolean doRun = false;
- private Thread selThread;
- private long selTimeout = 1000;
- private volatile Exception selEx;
-
- private Map<ConnectionInfo, ChannelConfiguration> channels;
- private Map<SelectableChannel, ChannelConfiguration> channelConfigurations;
-
- public static NetworkManager getInstance() throws IOException
- {
- if (_instance == null) {
- synchronized (NetworkManager.class) {
- if (_instance == null) {
- _instance = new NetworkManager();
- }
- }
- }
- return _instance;
- }
-
- private NetworkManager() throws IOException
- {
- channels = new HashMap<ConnectionInfo, ChannelConfiguration>();
- channelConfigurations = new HashMap<SelectableChannel, ChannelConfiguration>();
- }
-
- public synchronized <T extends SelectableChannel> ChannelAction<T> registerAction(int port, ConnectionType type, ChannelListener<T> listener, int ops) throws IOException
- {
- boolean startProc = (channels.size() == 0);
- SelectableChannel channel = null;
- SocketAddress address = new InetSocketAddress(port);
- ConnectionInfo connectionInfo = new ConnectionInfo();
- connectionInfo.address = address;
- connectionInfo.connectionType = type;
- ChannelConfiguration channelConfiguration = channels.get(connectionInfo);
- if (channelConfiguration == null) {
- Object socket = null;
- if (type == ConnectionType.TCP) {
- SocketChannel schannel = SocketChannel.open();
- schannel.configureBlocking(false);
- Socket ssocket = schannel.socket();
- ssocket.bind(address);
- socket = ssocket;
- channel = schannel;
- } else if (type == ConnectionType.UDP) {
- DatagramChannel dchannel = DatagramChannel.open();
- dchannel.configureBlocking(false);
- DatagramSocket dsocket = dchannel.socket();
- dsocket.bind(address);
- socket = dsocket;
- channel = dchannel;
- }
- if (channel == null) {
- throw new IOException("Unsupported connection type");
- }
- channelConfiguration = new ChannelConfiguration();
- channelConfiguration.actions = new ConcurrentLinkedQueue<ChannelAction>();
- channelConfiguration.channel = channel;
- channelConfiguration.connectionInfo = connectionInfo;
- channels.put(connectionInfo, channelConfiguration);
- channelConfigurations.put(channel, channelConfiguration);
- } else {
- channel = channelConfiguration.channel;
- }
- ChannelAction channelAction = new ChannelAction();
- channelAction.channelConfiguration = channelConfiguration;
- channelAction.listener = listener;
- channelAction.ops = ops;
- channelConfiguration.actions.add(channelAction);
- if (startProc) {
- startProcess();
- }
- if (listener != null) {
- channel.register(selector, ops);
- }
- return channelAction;
- }
-
- public synchronized void unregisterAction(ChannelAction action) throws IOException, InterruptedException
- {
- ChannelConfiguration channelConfiguration = action.channelConfiguration;
- SelectableChannel channel = channelConfiguration.channel;
- if (channelConfiguration != null) {
- channelConfiguration.actions.remove(action);
- if (channelConfiguration.actions.size() == 0) {
- ConnectionInfo connectionInfo = channelConfiguration.connectionInfo;
- channelConfigurations.remove(channel);
- channels.remove(connectionInfo);
- channel.close();
- }
- }
- if (channels.size() == 0) {
- stopProcess();
- }
- }
-
- private void startProcess() throws IOException
- {
- selector = Selector.open();
- doRun = true;
- selThread = new Thread(this);
- selThread.start();
- }
-
- private void stopProcess() throws InterruptedException, IOException
- {
- doRun = false;
- selThread.join();
- selector.close();
- }
-
- @Override
- public void run()
- {
- try {
- while (doRun) {
- int keys = selector.select(selTimeout);
- if (keys > 0) {
- Set<SelectionKey> selectionKeys = selector.selectedKeys();
- for (SelectionKey selectionKey : selectionKeys) {
- int readyOps = selectionKey.readyOps();
- ChannelConfiguration channelConfiguration = channelConfigurations.get(selectionKey.channel());
- Collection<ChannelAction> actions = channelConfiguration.actions;
- for (ChannelAction action : actions) {
- if (((readyOps & action.ops) != 0) && (action.listener != null)) {
- action.listener.ready(action, readyOps);
- }
- }
- }
- selectionKeys.clear();
- }
- }
- } catch (IOException e) {
- logger.error("Error in select", e);
- selEx = e;
- }
- }
-
- public static interface ChannelListener<T extends SelectableChannel>
- {
- public void ready(ChannelAction<T> action, int readyOps);
- }
-
- public static class ChannelConfiguration<T extends SelectableChannel>
- {
- public T channel;
- public ConnectionInfo connectionInfo;
- public Collection<ChannelAction> actions;
- }
-
- public static class ChannelAction<T extends SelectableChannel>
- {
- public ChannelConfiguration<T> channelConfiguration;
- public ChannelListener<T> listener;
- public int ops;
- }
-
- private static class ConnectionInfo
- {
- public SocketAddress address;
- public ConnectionType connectionType;
-
- @Override
- public boolean equals(Object o)
- {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
-
- ConnectionInfo that = (ConnectionInfo)o;
-
- if (connectionType != that.connectionType) {
- return false;
- }
- if (!address.equals(that.address)) {
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode()
- {
- int result = address.hashCode();
- result = 31 * result + connectionType.hashCode();
- return result;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/echoserver/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/demos/echoserver/src/main/resources/META-INF/properties.xml b/demos/echoserver/src/main/resources/META-INF/properties.xml
deleted file mode 100644
index 0822f4f..0000000
--- a/demos/echoserver/src/main/resources/META-INF/properties.xml
+++ /dev/null
@@ -1,38 +0,0 @@
-<?xml version="1.0"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<configuration>
- <!--
- <property>
- <name>dt.application.{appName}.operator.{opName}.prop.{propName}</name>
- <value>some-default-value (if value is not specified, it is required from the user or custom config when launching)</value>
- </property>
- -->
- <property>
- <name>dt.attr.MASTER_MEMORY_MB</name>
- <value>1024</value>
- </property>
- <property>
- <name>dt.application.MyFirstApplication.operator.console.prop.stringFormat</name>
- <value>hello world: %s</value>
- </property>
-</configuration>
-
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/echoserver/src/site/conf/my-app-conf1.xml
----------------------------------------------------------------------
diff --git a/demos/echoserver/src/site/conf/my-app-conf1.xml b/demos/echoserver/src/site/conf/my-app-conf1.xml
deleted file mode 100644
index f35873b..0000000
--- a/demos/echoserver/src/site/conf/my-app-conf1.xml
+++ /dev/null
@@ -1,27 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<configuration>
- <property>
- <name>dt.attr.MASTER_MEMORY_MB</name>
- <value>1024</value>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/echoserver/src/test/java/com/datatorrent/demos/echoserver/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/demos/echoserver/src/test/java/com/datatorrent/demos/echoserver/ApplicationTest.java b/demos/echoserver/src/test/java/com/datatorrent/demos/echoserver/ApplicationTest.java
deleted file mode 100644
index 8c52a9d..0000000
--- a/demos/echoserver/src/test/java/com/datatorrent/demos/echoserver/ApplicationTest.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.echoserver;
-
-import java.io.IOException;
-
-import javax.validation.ConstraintViolationException;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.LocalMode;
-
-/**
- * Test the DAG declaration in local mode.
- */
-public class ApplicationTest
-{
-
- @Test
- public void testApplication() throws IOException, Exception
- {
- try {
- LocalMode lma = LocalMode.newInstance();
- Configuration conf = new Configuration(false);
- conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
- lma.prepareDAG(new Application(), conf);
- LocalMode.Controller lc = lma.getController();
- lc.run(10000); // runs for 10 seconds and quits
- } catch (ConstraintViolationException e) {
- Assert.fail("constraint violations: " + e.getConstraintViolations());
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/echoserver/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/demos/echoserver/src/test/resources/log4j.properties b/demos/echoserver/src/test/resources/log4j.properties
deleted file mode 100644
index cf0d19e..0000000
--- a/demos/echoserver/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,43 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-log4j.rootLogger=DEBUG,CONSOLE
-
-log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
-log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
-log4j.appender.CONSOLE.threshold=${test.log.console.threshold}
-test.log.console.threshold=DEBUG
-
-log4j.appender.RFA=org.apache.log4j.RollingFileAppender
-log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
-log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
-log4j.appender.RFA.File=/tmp/app.log
-
-# to enable, add SYSLOG to rootLogger
-log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender
-log4j.appender.SYSLOG.syslogHost=127.0.0.1
-log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout
-log4j.appender.SYSLOG.layout.conversionPattern=${dt.cid} %-5p [%t] %c{2} %x - %m%n
-log4j.appender.SYSLOG.Facility=LOCAL1
-
-log4j.logger.org=info
-#log4j.logger.org.apache.commons.beanutils=warn
-log4j.logger.com.datatorrent=debug
-log4j.logger.org.apache.apex=debug
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/frauddetect/pom.xml
----------------------------------------------------------------------
diff --git a/demos/frauddetect/pom.xml b/demos/frauddetect/pom.xml
deleted file mode 100644
index 5900354..0000000
--- a/demos/frauddetect/pom.xml
+++ /dev/null
@@ -1,49 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>frauddetect-demo</artifactId>
- <packaging>jar</packaging>
-
- <name>Apache Apex Malhar Fraud Detect Demo</name>
- <description>Apex demo application that demonstrates real-time pattern detection in the incoming data and alerting. The demo processes streaming credit card transactions and looks for fraudulent transactions.</description>
-
- <parent>
- <groupId>org.apache.apex</groupId>
- <artifactId>malhar-demos</artifactId>
- <version>3.7.0-SNAPSHOT</version>
- </parent>
-
- <properties>
- <skipTests>true</skipTests>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>org.mongodb</groupId>
- <artifactId>mongo-java-driver</artifactId>
- <version>2.10.1</version>
- </dependency>
- </dependencies>
-
-</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/frauddetect/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/demos/frauddetect/src/assemble/appPackage.xml b/demos/frauddetect/src/assemble/appPackage.xml
deleted file mode 100644
index 4138cf2..0000000
--- a/demos/frauddetect/src/assemble/appPackage.xml
+++ /dev/null
@@ -1,59 +0,0 @@
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
- <id>appPackage</id>
- <formats>
- <format>jar</format>
- </formats>
- <includeBaseDirectory>false</includeBaseDirectory>
- <fileSets>
- <fileSet>
- <directory>${basedir}/target/</directory>
- <outputDirectory>/app</outputDirectory>
- <includes>
- <include>${project.artifactId}-${project.version}.jar</include>
- </includes>
- </fileSet>
- <fileSet>
- <directory>${basedir}/target/deps</directory>
- <outputDirectory>/lib</outputDirectory>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/site/conf</directory>
- <outputDirectory>/conf</outputDirectory>
- <includes>
- <include>*.xml</include>
- </includes>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/main/resources/META-INF</directory>
- <outputDirectory>/META-INF</outputDirectory>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/main/resources/app</directory>
- <outputDirectory>/app</outputDirectory>
- </fileSet>
- </fileSets>
-
-</assembly>
-
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/Application.java
----------------------------------------------------------------------
diff --git a/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/Application.java b/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/Application.java
deleted file mode 100644
index 8d7c325..0000000
--- a/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/Application.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.frauddetect;
-
-import java.io.Serializable;
-import java.net.URI;
-import org.apache.hadoop.conf.Configuration;
-import com.datatorrent.api.Context;
-import com.datatorrent.api.Context.DAGContext;
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-import com.datatorrent.demos.frauddetect.operator.HdfsStringOutputOperator;
-import com.datatorrent.demos.frauddetect.operator.MongoDBOutputOperator;
-import com.datatorrent.lib.io.ConsoleOutputOperator;
-import com.datatorrent.lib.io.PubSubWebSocketInputOperator;
-import com.datatorrent.lib.io.PubSubWebSocketOutputOperator;
-import com.datatorrent.lib.math.RangeKeyVal;
-import com.datatorrent.lib.multiwindow.SimpleMovingAverage;
-import com.datatorrent.lib.util.BaseKeyValueOperator;
-import com.datatorrent.lib.util.KeyValPair;
-import com.datatorrent.netlet.util.DTThrowable;
-
-
-/**
- * Fraud detection application
- *
- * @since 0.9.0
- */
-@ApplicationAnnotation(name = "FraudDetectDemo")
-public class Application implements StreamingApplication
-{
-
- public PubSubWebSocketInputOperator getPubSubWebSocketInputOperator(String name, DAG dag, URI duri, String topic) throws Exception
- {
- PubSubWebSocketInputOperator reqin = dag.addOperator(name, new PubSubWebSocketInputOperator());
- reqin.setUri(duri);
- reqin.setTopic(topic);
- return reqin;
- }
-
- public PubSubWebSocketOutputOperator getPubSubWebSocketOutputOperator(String name, DAG dag, URI duri, String topic) throws Exception
- {
- PubSubWebSocketOutputOperator out = dag.addOperator(name, new PubSubWebSocketOutputOperator());
- out.setUri(duri);
- return out;
- }
-
- public HdfsStringOutputOperator getHdfsOutputOperator(String name, DAG dag, String folderName)
- {
- HdfsStringOutputOperator oper = dag.addOperator("hdfs", HdfsStringOutputOperator.class);
- oper.setFilePath(folderName);
- oper.setMaxLength(1024 * 1024 * 1024);
- return oper;
- }
-
- public ConsoleOutputOperator getConsoleOperator(String name, DAG dag, String prefix, String format)
- {
- ConsoleOutputOperator oper = dag.addOperator(name, ConsoleOutputOperator.class);
- // oper.setStringFormat(prefix + ": " + format);
- return oper;
- }
-
- public static class KeyPartitionCodec<K, V> extends BaseKeyValueOperator.DefaultPartitionCodec<K,V> implements Serializable
- {
- private static final long serialVersionUID = 201410031623L;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
-
- try {
- String gatewayAddress = dag.getValue(DAGContext.GATEWAY_CONNECT_ADDRESS);
- if (gatewayAddress == null) {
- gatewayAddress = "localhost:9090";
- }
- URI duri = URI.create("ws://" + gatewayAddress + "/pubsub");
-
- PubSubWebSocketInputOperator userTxWsInput = getPubSubWebSocketInputOperator("userTxInput", dag, duri, "demos.app.frauddetect.submitTransaction");
- PubSubWebSocketOutputOperator ccUserAlertWsOutput = getPubSubWebSocketOutputOperator("ccUserAlertQueryOutput", dag, duri, "demos.app.frauddetect.fraudAlert");
- PubSubWebSocketOutputOperator avgUserAlertwsOutput = getPubSubWebSocketOutputOperator("avgUserAlertQueryOutput", dag, duri, "demos.app.frauddetect.fraudAlert");
- PubSubWebSocketOutputOperator binUserAlertwsOutput = getPubSubWebSocketOutputOperator("binUserAlertOutput", dag, duri, "demos.app.frauddetect.fraudAlert");
- PubSubWebSocketOutputOperator txSummaryWsOutput = getPubSubWebSocketOutputOperator("txSummaryWsOutput", dag, duri, "demos.app.frauddetect.txSummary");
- SlidingWindowSumKeyVal<KeyValPair<MerchantKey, String>, Integer> smsOperator = dag.addOperator("movingSum", SlidingWindowSumKeyVal.class);
-
- MerchantTransactionGenerator txReceiver = dag.addOperator("txReceiver", MerchantTransactionGenerator.class);
- MerchantTransactionInputHandler txInputHandler = dag.addOperator("txInputHandler", new MerchantTransactionInputHandler());
- BankIdNumberSamplerOperator binSampler = dag.addOperator("bankInfoFraudDetector", BankIdNumberSamplerOperator.class);
-
- MerchantTransactionBucketOperator txBucketOperator = dag.addOperator("txFilter", MerchantTransactionBucketOperator.class);
- RangeKeyVal rangeOperator = dag.addOperator("rangePerMerchant", new RangeKeyVal<MerchantKey, Long>());
- SimpleMovingAverage<MerchantKey, Long> smaOperator = dag.addOperator("smaPerMerchant", SimpleMovingAverage.class);
- TransactionStatsAggregator txStatsAggregator = dag.addOperator("txStatsAggregator", TransactionStatsAggregator.class);
- AverageAlertingOperator avgAlertingOperator = dag.addOperator("avgAlerter", AverageAlertingOperator.class);
- CreditCardAmountSamplerOperator ccSamplerOperator = dag.addOperator("amountFraudDetector", CreditCardAmountSamplerOperator.class);
- HdfsStringOutputOperator hdfsOutputOperator = getHdfsOutputOperator("hdfsOutput", dag, "fraud");
-
- MongoDBOutputOperator mongoTxStatsOperator = dag.addOperator("mongoTxStatsOutput", MongoDBOutputOperator.class);
- MongoDBOutputOperator mongoBinAlertsOperator = dag.addOperator("mongoBinAlertsOutput", MongoDBOutputOperator.class);
- MongoDBOutputOperator mongoCcAlertsOperator = dag.addOperator("mongoCcAlertsOutput", MongoDBOutputOperator.class);
- MongoDBOutputOperator mongoAvgAlertsOperator = dag.addOperator("mongoAvgAlertsOutput", MongoDBOutputOperator.class);
-
- dag.addStream("userTxStream", userTxWsInput.outputPort, txInputHandler.userTxInputPort);
- dag.addStream("transactions", txReceiver.txOutputPort, txBucketOperator.inputPort).setLocality(DAG.Locality.CONTAINER_LOCAL);
- dag.addStream("txData", txReceiver.txDataOutputPort, hdfsOutputOperator.input); // dump all tx into Hdfs
- dag.addStream("userTransactions", txInputHandler.txOutputPort, txBucketOperator.txUserInputPort);
- dag.addStream("bankInfoData", txBucketOperator.binCountOutputPort, smsOperator.data);
- dag.addStream("bankInfoCount", smsOperator.integerSum, binSampler.txCountInputPort);
- dag.addStream("filteredTransactions", txBucketOperator.txOutputPort, rangeOperator.data, smaOperator.data, avgAlertingOperator.txInputPort);
-
- KeyPartitionCodec<MerchantKey, Long> txCodec = new KeyPartitionCodec<MerchantKey, Long>();
- dag.setInputPortAttribute(rangeOperator.data, Context.PortContext.STREAM_CODEC, txCodec);
- dag.setInputPortAttribute(smaOperator.data, Context.PortContext.STREAM_CODEC, txCodec);
- dag.setInputPortAttribute(avgAlertingOperator.txInputPort, Context.PortContext.STREAM_CODEC, txCodec);
-
- dag.addStream("creditCardData", txBucketOperator.ccAlertOutputPort, ccSamplerOperator.inputPort);
- dag.addStream("txnSummaryData", txBucketOperator.summaryTxnOutputPort, txSummaryWsOutput.input);
- dag.addStream("smaAlerts", smaOperator.doubleSMA, avgAlertingOperator.smaInputPort);
- dag.addStream("binAlerts", binSampler.countAlertOutputPort, mongoBinAlertsOperator.inputPort);
- dag.addStream("binAlertsNotification", binSampler.countAlertNotificationPort, binUserAlertwsOutput.input);
- dag.addStream("rangeData", rangeOperator.range, txStatsAggregator.rangeInputPort);
- dag.addStream("smaData", smaOperator.longSMA, txStatsAggregator.smaInputPort);
- dag.addStream("txStatsOutput", txStatsAggregator.txDataOutputPort, mongoTxStatsOperator.inputPort);
- dag.addStream("avgAlerts", avgAlertingOperator.avgAlertOutputPort, mongoAvgAlertsOperator.inputPort);
- dag.addStream("avgAlertsNotification", avgAlertingOperator.avgAlertNotificationPort, avgUserAlertwsOutput.input);
- dag.addStream("ccAlerts", ccSamplerOperator.ccAlertOutputPort, mongoCcAlertsOperator.inputPort);
- dag.addStream("ccAlertsNotification", ccSamplerOperator.ccAlertNotificationPort, ccUserAlertWsOutput.input);
-
- } catch (Exception exc) {
- DTThrowable.rethrow(exc);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/AverageAlertData.java
----------------------------------------------------------------------
diff --git a/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/AverageAlertData.java b/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/AverageAlertData.java
deleted file mode 100644
index 961be96..0000000
--- a/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/AverageAlertData.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.frauddetect;
-
-/**
- * POJO to capture average alert data.
- *
- * @since 0.9.0
- */
-public class AverageAlertData
-{
- public String merchantId;
- public int terminalId;
- public int zipCode;
- public MerchantTransaction.MerchantType merchantType;
- public long amount;
- public double lastSmaValue;
- public double change;
- public boolean userGenerated;
- public long time;
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/AverageAlertingOperator.java
----------------------------------------------------------------------
diff --git a/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/AverageAlertingOperator.java b/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/AverageAlertingOperator.java
deleted file mode 100644
index b813a40..0000000
--- a/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/AverageAlertingOperator.java
+++ /dev/null
@@ -1,185 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.frauddetect;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import javax.validation.constraints.NotNull;
-
-import org.codehaus.jackson.JsonFactory;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.commons.lang.mutable.MutableDouble;
-
-import com.datatorrent.api.DefaultInputPort;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.common.util.BaseOperator;
-import com.datatorrent.demos.frauddetect.util.JsonUtils;
-import com.datatorrent.lib.util.KeyValPair;
-
-/**
- * Generate an alert if the current transaction amount received on tx input port for the given key is greater by n %
- * than the SMA of the last application window as received on the SMA input port.
- *
- * @since 0.9.0
- */
-public class AverageAlertingOperator extends BaseOperator
-{
- private static final Logger Log = LoggerFactory.getLogger(AverageAlertingOperator.class);
- private final transient JsonFactory jsonFactory = new JsonFactory();
- private final transient ObjectMapper mapper = new ObjectMapper(jsonFactory);
- private Map<MerchantKey, MutableDouble> lastSMAMap = new HashMap<MerchantKey, MutableDouble>();
- private Map<MerchantKey, MutableDouble> currentSMAMap = new HashMap<MerchantKey, MutableDouble>();
- private List<AverageAlertData> alerts = new ArrayList<AverageAlertData>();
- @NotNull
- private int threshold;
- private static final String brickMortarAlertMsg = "Transaction amount %d exceeded by %f (last SMA %f) for Merchant %s at Terminal %d!";
- private static final String internetAlertMsg = "Transaction amount %d exceeded by %f (last SMA %f) for Merchant %s!";
- public final transient DefaultOutputPort<String> avgAlertOutputPort = new DefaultOutputPort<String>();
- public final transient DefaultOutputPort<Map<String, Object>> avgAlertNotificationPort = new DefaultOutputPort<Map<String, Object>>();
- public final transient DefaultInputPort<KeyValPair<MerchantKey, Double>> smaInputPort =
- new DefaultInputPort<KeyValPair<MerchantKey, Double>>()
- {
- @Override
- public void process(KeyValPair<MerchantKey, Double> tuple)
- {
- MutableDouble currentSma = currentSMAMap.get(tuple.getKey());
- if (currentSma == null) { // first sma for the given key
- double sma = tuple.getValue();
- currentSMAMap.put(tuple.getKey(), new MutableDouble(sma));
- //lastSMAMap.put(tuple.getKey(), new MutableDouble(sma));
- } else { // move the current SMA value to the last SMA Map
- //lastSMAMap.get(tuple.getKey()).setValue(currentSma.getValue());
- currentSma.setValue(tuple.getValue()); // update the current SMA value
- }
- }
-
- };
- public final transient DefaultInputPort<KeyValPair<MerchantKey, Long>> txInputPort =
- new DefaultInputPort<KeyValPair<MerchantKey, Long>>()
- {
- @Override
- public void process(KeyValPair<MerchantKey, Long> tuple)
- {
- processTuple(tuple);
- }
-
- };
-
- private void processTuple(KeyValPair<MerchantKey, Long> tuple)
- {
- MerchantKey merchantKey = tuple.getKey();
- MutableDouble lastSma = lastSMAMap.get(tuple.getKey());
- long txValue = tuple.getValue();
- if (lastSma != null && txValue > lastSma.doubleValue()) {
- double lastSmaValue = lastSma.doubleValue();
- double change = txValue - lastSmaValue;
- if (change > threshold) { // generate an alert
- AverageAlertData data = getOutputData(merchantKey, txValue, change, lastSmaValue);
- alerts.add(data);
- //if (userGenerated) { // if its user generated only the pass it to WebSocket
- if (merchantKey.merchantType == MerchantTransaction.MerchantType.BRICK_AND_MORTAR) {
- avgAlertNotificationPort.emit(getOutputData(data, String.format(brickMortarAlertMsg, txValue, change, lastSmaValue, merchantKey.merchantId, merchantKey.terminalId)));
- } else { // its internet based
- avgAlertNotificationPort.emit(getOutputData(data, String.format(internetAlertMsg, txValue, change, lastSmaValue, merchantKey.merchantId)));
-
- }
- //}
- }
- }
- }
-
- @Override
- public void endWindow()
- {
- for (AverageAlertData data : alerts) {
- try {
- avgAlertOutputPort.emit(JsonUtils.toJson(data));
- } catch (IOException e) {
- logger.warn("Exception while converting object to JSON", e);
- }
- }
-
- alerts.clear();
-
- for (Map.Entry<MerchantKey, MutableDouble> entry : currentSMAMap.entrySet()) {
- MerchantKey key = entry.getKey();
- MutableDouble currentSma = entry.getValue();
- MutableDouble lastSma = lastSMAMap.get(key);
- if (lastSma == null) {
- lastSma = new MutableDouble(currentSma.doubleValue());
- lastSMAMap.put(key, lastSma);
- } else {
- lastSma.setValue(currentSma.getValue());
- }
- }
- }
-
- private AverageAlertData getOutputData(MerchantKey key, long amount, double change, double lastSmaValue)
- {
- AverageAlertData data = new AverageAlertData();
-
- data.merchantId = key.merchantId;
- data.terminalId = key.terminalId == null ? 0 : key.terminalId;
- data.zipCode = key.zipCode;
- data.merchantType = key.merchantType;
- data.amount = amount;
- data.lastSmaValue = lastSmaValue;
- data.change = change;
- //data.userGenerated = userGenerated;
- data.userGenerated = key.userGenerated;
- data.time = System.currentTimeMillis();
-
- return data;
- }
-
- private Map<String, Object> getOutputData(AverageAlertData data, String msg)
- {
- Map<String, Object> output = new HashMap<String, Object>();
- output.put("message", msg);
- output.put("alertType", "aboveAvg");
- output.put("userGenerated", "" + data.userGenerated);
- output.put("alertData", data);
-
- try {
- String str = mapper.writeValueAsString(output);
- logger.debug("user generated tx alert: " + str);
- } catch (Exception exc) {
- //ignore
- }
- return output;
- }
-
- public int getThreshold()
- {
- return threshold;
- }
-
- public void setThreshold(int threshold)
- {
- this.threshold = threshold;
- }
-
- private static final Logger logger = LoggerFactory.getLogger(AverageAlertingOperator.class);
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/BankIdNumberAlertData.java
----------------------------------------------------------------------
diff --git a/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/BankIdNumberAlertData.java b/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/BankIdNumberAlertData.java
deleted file mode 100644
index 6f18199..0000000
--- a/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/BankIdNumberAlertData.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.frauddetect;
-
-/**
- * POJO to capture data related to alerts for repetitive bank id number data usage.
- *
- * @since 0.9.0
- */
-public class BankIdNumberAlertData
-{
- public String merchantId;
- public int terminalId;
- public int zipCode;
- public MerchantTransaction.MerchantType merchantType;
- public String bankIdNum;
- public int count;
- public boolean userGenerated;
- public long time;
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/BankIdNumberKey.java
----------------------------------------------------------------------
diff --git a/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/BankIdNumberKey.java b/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/BankIdNumberKey.java
deleted file mode 100644
index 87cf043..0000000
--- a/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/BankIdNumberKey.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.frauddetect;
-
-import java.io.Serializable;
-
-import com.datatorrent.lib.util.TimeBucketKey;
-
-/**
- * Bank Id Number Key
- *
- * @since 0.9.0
- */
-public class BankIdNumberKey extends TimeBucketKey implements Serializable
-{
- public String bankIdNum;
-
- public BankIdNumberKey()
- {
- }
-
- @Override
- public int hashCode()
- {
- int key = 0;
- key |= (1 << 1);
- key |= (bankIdNum.hashCode());
- return super.hashCode() ^ key;
- }
-
- @Override
- public boolean equals(Object obj)
- {
- if (!(obj instanceof BankIdNumberKey)) {
- return false;
- }
- return super.equals(obj)
- && bankIdNum.equals(((BankIdNumberKey)obj).bankIdNum);
- }
-
- @Override
- public String toString()
- {
- StringBuilder sb = new StringBuilder(super.toString());
- sb.append("|1:").append(bankIdNum);
- return sb.toString();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/BankIdNumberSamplerOperator.java
----------------------------------------------------------------------
diff --git a/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/BankIdNumberSamplerOperator.java b/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/BankIdNumberSamplerOperator.java
deleted file mode 100644
index abfa202..0000000
--- a/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/BankIdNumberSamplerOperator.java
+++ /dev/null
@@ -1,216 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.frauddetect;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.codehaus.jackson.JsonFactory;
-import org.codehaus.jackson.map.ObjectMapper;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.commons.lang.mutable.MutableLong;
-
-import com.datatorrent.api.DefaultInputPort;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.common.util.BaseOperator;
-import com.datatorrent.demos.frauddetect.util.JsonUtils;
-import com.datatorrent.lib.util.KeyValPair;
-
-/**
- * Count the transactions for the underlying aggregation window if the same BIN is
- * being used for more than defined number of transactions. Output the data as needed
- * by Mongo output operator
- *
- * @since 0.9.0
- */
-public class BankIdNumberSamplerOperator extends BaseOperator
-{
- private final transient JsonFactory jsonFactory = new JsonFactory();
- private final transient ObjectMapper mapper = new ObjectMapper(jsonFactory);
- private int threshold;
- private Map<MerchantKey, Map<String, BankIdNumData>> bankIdNumCountMap = new HashMap<MerchantKey, Map<String, BankIdNumData>>();
- private static final String ALERT_MSG =
- "Potential fraudulent CC transactions (same bank id %s and merchant %s) total transactions: %d";
- /**
- * Output the key-value pair for the BIN as key with the count as value.
- */
- public final transient DefaultOutputPort<String> countAlertOutputPort =
- new DefaultOutputPort<String>();
- public final transient DefaultOutputPort<Map<String, Object>> countAlertNotificationPort =
- new DefaultOutputPort<Map<String, Object>>();
-
- public int getThreshold()
- {
- return threshold;
- }
-
- public void setThreshold(int threshold)
- {
- this.threshold = threshold;
- }
-
- /*
- public final transient DefaultInputPort<KeyValPair<MerchantKey, String>> txInputPort =
- new DefaultInputPort<KeyValPair<MerchantKey, String>>()
- {
- @Override
- public void process(KeyValPair<MerchantKey, String> tuple)
- {
- processTuple(tuple);
- }
-
- };
-
- private void processTuple(KeyValPair<MerchantKey, String> tuple)
- {
- MerchantKey key = tuple.getKey();
- Map<String, BankIdNumData> map = bankIdNumCountMap.get(key);
- if (map == null) {
- map = new HashMap<String, BankIdNumData>();
- bankIdNumCountMap.put(key, map);
- }
- String bankIdNum = tuple.getValue();
- BankIdNumData bankIdNumData = map.get(bankIdNum);
- if (bankIdNumData == null) {
- bankIdNumData = new BankIdNumData();
- bankIdNumData.bankIdNum = bankIdNum;
- map.put(bankIdNum, bankIdNumData);
- }
- bankIdNumData.count.increment();
- if (key.userGenerated) {
- bankIdNumData.userGenerated = true;
- }
- }
- */
-
- public final transient DefaultInputPort<KeyValPair<KeyValPair<MerchantKey, String>, Integer>> txCountInputPort =
- new DefaultInputPort<KeyValPair<KeyValPair<MerchantKey, String>, Integer>>()
- {
- @Override
- public void process(KeyValPair<KeyValPair<MerchantKey, String>, Integer> tuple)
- {
- processTuple(tuple.getKey(), tuple.getValue());
- }
-
- };
-
- private void processTuple(KeyValPair<MerchantKey, String> tuple, Integer count)
- {
- MerchantKey key = tuple.getKey();
- Map<String, BankIdNumData> map = bankIdNumCountMap.get(key);
- if (map == null) {
- map = new HashMap<String, BankIdNumData>();
- bankIdNumCountMap.put(key, map);
- }
- String bankIdNum = tuple.getValue();
- BankIdNumData bankIdNumData = map.get(bankIdNum);
- if (bankIdNumData == null) {
- bankIdNumData = new BankIdNumData();
- bankIdNumData.bankIdNum = bankIdNum;
- map.put(bankIdNum, bankIdNumData);
- }
- bankIdNumData.count.setValue(count);
- if (key.userGenerated) {
- bankIdNumData.userGenerated = true;
- }
- }
-
- /**
- * Go through the BIN Counter map and check if any of the values for the BIN exceed the threshold.
- * If yes, generate the alert on the output port.
- */
- @Override
- public void endWindow()
- {
- for (Map.Entry<MerchantKey, Map<String, BankIdNumData>> entry : bankIdNumCountMap.entrySet()) {
- List<BankIdNumData> list = null;
- MerchantKey key = entry.getKey();
- if (key.merchantType == MerchantTransaction.MerchantType.INTERNET) {
- continue;
- }
- list = dataOutput(entry.getValue());
- if (list.size() > 0) {
- for (BankIdNumData binData : list) {
- BankIdNumberAlertData data = new BankIdNumberAlertData();
- data.merchantId = key.merchantId;
- data.terminalId = key.terminalId == null ? 0 : key.terminalId;
- data.zipCode = key.zipCode;
- data.merchantType = key.merchantType;
- data.bankIdNum = binData.bankIdNum;
- data.count = binData.count.intValue();
- data.userGenerated = binData.userGenerated;
- data.time = System.currentTimeMillis();
- try {
- countAlertOutputPort.emit(JsonUtils.toJson(data));
- countAlertNotificationPort.emit(getOutputData(data));
- } catch (IOException e) {
- logger.warn("Exception while converting object to JSON: ", e);
- }
- }
- }
- }
- bankIdNumCountMap.clear();
- }
-
- private List<BankIdNumData> dataOutput(Map<String, BankIdNumData> map)
- {
- List<BankIdNumData> list = new ArrayList<BankIdNumData>();
- int count = 0;
- for (Map.Entry<String, BankIdNumData> bankIdEntry : map.entrySet()) {
- BankIdNumData data = bankIdEntry.getValue();
- if (data.count.intValue() > threshold) {
- list.add(data);
- }
- }
- return list;
- }
-
- private Map<String, Object> getOutputData(BankIdNumberAlertData data)
- {
- Map<String, Object> output = new HashMap<String, Object>();
- output.put("message", String.format(ALERT_MSG, data.bankIdNum, data.merchantId, data.count));
- output.put("alertType", "sameBankId");
- output.put("userGenerated", "" + data.userGenerated);
- output.put("alertData", data);
-
- try {
- String str = mapper.writeValueAsString(output);
- logger.debug("user generated tx alert: " + str);
- } catch (Exception exc) {
- //ignore
- }
-
- return output;
- }
-
- public static final class BankIdNumData
- {
- public String bankIdNum;
- public MutableLong count = new MutableLong();
- public boolean userGenerated = false;
- }
-
- private static final Logger logger = LoggerFactory.getLogger(BankIdNumberSamplerOperator.class);
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/CreditCardAlertData.java
----------------------------------------------------------------------
diff --git a/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/CreditCardAlertData.java b/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/CreditCardAlertData.java
deleted file mode 100644
index 885bf94..0000000
--- a/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/CreditCardAlertData.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.frauddetect;
-
-/**
- * POJO to capture data related to alerts for credit card number.
- *
- * @since 0.9.0
- */
-public class CreditCardAlertData
-{
- public String merchantId;
- public int terminalId;
- public int zipCode;
- public MerchantTransaction.MerchantType merchantType;
- public String fullCcNum;
- public long small;
- public long large;
- public double threshold;
- public boolean userGenerated;
- public long time;
-}
[20/30] apex-malhar git commit: Renamed demos to examples. Packages
and artifactid names are changed as suggested.
Posted by th...@apache.org.
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/OutputCollectorImpl.java
----------------------------------------------------------------------
diff --git a/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/OutputCollectorImpl.java b/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/OutputCollectorImpl.java
deleted file mode 100644
index 6c81724..0000000
--- a/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/OutputCollectorImpl.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.mroperator;
-
-import java.io.IOException;
-import java.io.PipedInputStream;
-import java.io.PipedOutputStream;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.serializer.Deserializer;
-import org.apache.hadoop.io.serializer.SerializationFactory;
-import org.apache.hadoop.io.serializer.Serializer;
-import org.apache.hadoop.mapred.OutputCollector;
-import com.datatorrent.lib.util.KeyHashValPair;
-
-/**
- * <p>OutputCollectorImpl class.</p>
- *
- * @since 0.9.0
- */
-@SuppressWarnings("unchecked")
-public class OutputCollectorImpl<K extends Object, V extends Object> implements OutputCollector<K, V>
-{
- private static final Logger logger = LoggerFactory.getLogger(OutputCollectorImpl.class);
-
- private List<KeyHashValPair<K, V>> list = new ArrayList<KeyHashValPair<K, V>>();
-
- public List<KeyHashValPair<K, V>> getList()
- {
- return list;
- }
-
- private transient SerializationFactory serializationFactory;
- private transient Configuration conf = null;
-
- public OutputCollectorImpl()
- {
- conf = new Configuration();
- serializationFactory = new SerializationFactory(conf);
-
- }
-
- private <T> T cloneObj(T t) throws IOException
- {
- Serializer<T> keySerializer;
- Class<T> keyClass;
- PipedInputStream pis = new PipedInputStream();
- PipedOutputStream pos = new PipedOutputStream(pis);
- keyClass = (Class<T>)t.getClass();
- keySerializer = serializationFactory.getSerializer(keyClass);
- keySerializer.open(pos);
- keySerializer.serialize(t);
- Deserializer<T> keyDesiralizer = serializationFactory.getDeserializer(keyClass);
- keyDesiralizer.open(pis);
- T clonedArg0 = keyDesiralizer.deserialize(null);
- pos.close();
- pis.close();
- keySerializer.close();
- keyDesiralizer.close();
- return clonedArg0;
-
- }
-
- @Override
- public void collect(K arg0, V arg1) throws IOException
- {
- if (conf == null) {
- conf = new Configuration();
- serializationFactory = new SerializationFactory(conf);
- }
- list.add(new KeyHashValPair<K, V>(cloneObj(arg0), cloneObj(arg1)));
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/ReduceOperator.java
----------------------------------------------------------------------
diff --git a/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/ReduceOperator.java b/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/ReduceOperator.java
deleted file mode 100644
index 5df9b0d..0000000
--- a/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/ReduceOperator.java
+++ /dev/null
@@ -1,189 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.mroperator;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapred.Counters;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.OutputCollector;
-import org.apache.hadoop.mapred.Reducer;
-import org.apache.hadoop.mapred.Reporter;
-
-import com.datatorrent.api.Context.OperatorContext;
-import com.datatorrent.api.DefaultInputPort;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.api.Operator;
-import com.datatorrent.demos.mroperator.ReporterImpl.ReporterType;
-import com.datatorrent.lib.util.KeyHashValPair;
-
-/**
- * <p>ReduceOperator class.</p>
- *
- * @since 0.9.0
- */
-@SuppressWarnings({ "deprecation", "unused" })
-public class ReduceOperator<K1, V1, K2, V2> implements Operator
-{
- private static final Logger logger = LoggerFactory.getLogger(ReduceOperator.class);
-
- private Class<? extends Reducer<K1, V1, K2, V2>> reduceClass;
- private transient Reducer<K1, V1, K2, V2> reduceObj;
- private transient Reporter reporter;
- private OutputCollector<K2, V2> outputCollector;
- private String configFile;
-
- public Class<? extends Reducer<K1, V1, K2, V2>> getReduceClass()
- {
- return reduceClass;
- }
-
- public void setReduceClass(Class<? extends Reducer<K1, V1, K2, V2>> reduceClass)
- {
- this.reduceClass = reduceClass;
- }
-
- public String getConfigFile()
- {
- return configFile;
- }
-
- public void setConfigFile(String configFile)
- {
- this.configFile = configFile;
- }
-
- private int numberOfMappersRunning = -1;
- private int operatorId;
-
- public final transient DefaultInputPort<KeyHashValPair<Integer, Integer>> inputCount = new DefaultInputPort<KeyHashValPair<Integer, Integer>>()
- {
- @Override
- public void process(KeyHashValPair<Integer, Integer> tuple)
- {
- logger.info("processing {}", tuple);
- if (numberOfMappersRunning == -1) {
- numberOfMappersRunning = tuple.getValue();
- } else {
- numberOfMappersRunning += tuple.getValue();
- }
-
- }
-
- };
-
- public final transient DefaultOutputPort<KeyHashValPair<K2, V2>> output = new DefaultOutputPort<KeyHashValPair<K2, V2>>();
- private Map<K1, List<V1>> cacheObject;
- public final transient DefaultInputPort<KeyHashValPair<K1, V1>> input = new DefaultInputPort<KeyHashValPair<K1, V1>>()
- {
- @Override
- public void process(KeyHashValPair<K1, V1> tuple)
- {
- // logger.info("processing tupple {}",tuple);
- List<V1> list = cacheObject.get(tuple.getKey());
- if (list == null) {
- list = new ArrayList<V1>();
- list.add(tuple.getValue());
- cacheObject.put(tuple.getKey(), list);
- } else {
- list.add(tuple.getValue());
- }
- }
-
- };
-
- @Override
- public void setup(OperatorContext context)
- {
- reporter = new ReporterImpl(ReporterType.Reducer, new Counters());
- if (context != null) {
- operatorId = context.getId();
- }
- cacheObject = new HashMap<K1, List<V1>>();
- outputCollector = new OutputCollectorImpl<K2, V2>();
- if (reduceClass != null) {
- try {
- reduceObj = reduceClass.newInstance();
- } catch (Exception e) {
- logger.info("can't instantiate object {}", e.getMessage());
- throw new RuntimeException(e);
- }
- Configuration conf = new Configuration();
- InputStream stream = null;
- if (configFile != null && configFile.length() > 0) {
- logger.info("system /{}", configFile);
- stream = ClassLoader.getSystemResourceAsStream("/" + configFile);
- if (stream == null) {
- logger.info("system {}", configFile);
- stream = ClassLoader.getSystemResourceAsStream(configFile);
- }
- }
- if (stream != null) {
- logger.info("found our stream... so adding it");
- conf.addResource(stream);
- }
- reduceObj.configure(new JobConf(conf));
- }
-
- }
-
- @Override
- public void teardown()
- {
-
- }
-
- @Override
- public void beginWindow(long windowId)
- {
-
- }
-
- @Override
- public void endWindow()
- {
- if (numberOfMappersRunning == 0) {
- for (Map.Entry<K1, List<V1>> e : cacheObject.entrySet()) {
- try {
- reduceObj.reduce(e.getKey(), e.getValue().iterator(), outputCollector, reporter);
- } catch (IOException e1) {
- logger.info(e1.getMessage());
- throw new RuntimeException(e1);
- }
- }
- List<KeyHashValPair<K2, V2>> list = ((OutputCollectorImpl<K2, V2>)outputCollector).getList();
- for (KeyHashValPair<K2, V2> e : list) {
- output.emit(e);
- }
- list.clear();
- cacheObject.clear();
- numberOfMappersRunning = -1;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/ReporterImpl.java
----------------------------------------------------------------------
diff --git a/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/ReporterImpl.java b/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/ReporterImpl.java
deleted file mode 100644
index d2d38da..0000000
--- a/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/ReporterImpl.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.mroperator;
-
-import org.apache.hadoop.mapred.Counters;
-import org.apache.hadoop.mapred.Counters.Counter;
-import org.apache.hadoop.mapred.InputSplit;
-import org.apache.hadoop.mapred.Reporter;
-
-/**
- * <p>ReporterImpl class.</p>
- *
- * @since 0.9.0
- */
-public class ReporterImpl implements Reporter
-{
- private Counters counters;
- InputSplit inputSplit;
-
- public enum ReporterType
- {
- Mapper, Reducer
- }
-
- private ReporterType typ;
-
- public ReporterImpl(final ReporterType kind, final Counters ctrs)
- {
- this.typ = kind;
- this.counters = ctrs;
- }
-
- @Override
- public InputSplit getInputSplit()
- {
- if (typ == ReporterType.Reducer) {
- throw new UnsupportedOperationException("Reducer cannot call getInputSplit()");
- } else {
- return inputSplit;
- }
- }
-
- public void setInputSplit(InputSplit inputSplit)
- {
- this.inputSplit = inputSplit;
- }
-
- @Override
- public void incrCounter(Enum<?> key, long amount)
- {
- if (null != counters) {
- counters.incrCounter(key, amount);
- }
- }
-
- @Override
- public void incrCounter(String group, String counter, long amount)
- {
- if (null != counters) {
- counters.incrCounter(group, counter, amount);
- }
- }
-
- @Override
- public void setStatus(String status)
- {
- // do nothing.
- }
-
- @Override
- public void progress()
- {
- // do nothing.
- }
-
- @Override
- public Counter getCounter(String group, String name)
- {
- Counters.Counter counter = null;
- if (counters != null) {
- counter = counters.findCounter(group, name);
- }
-
- return counter;
- }
-
- @Override
- public Counter getCounter(Enum<?> key)
- {
- Counters.Counter counter = null;
- if (counters != null) {
- counter = counters.findCounter(key);
- }
-
- return counter;
- }
-
- public float getProgress()
- {
- return 0;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/WordCount.java
----------------------------------------------------------------------
diff --git a/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/WordCount.java b/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/WordCount.java
deleted file mode 100644
index f78cf99..0000000
--- a/demos/mroperator/src/main/java/com/datatorrent/demos/mroperator/WordCount.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.mroperator;
-
-import java.io.IOException;
-import java.util.Iterator;
-import java.util.StringTokenizer;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.FileInputFormat;
-import org.apache.hadoop.mapred.FileOutputFormat;
-import org.apache.hadoop.mapred.JobClient;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.MapReduceBase;
-import org.apache.hadoop.mapred.Mapper;
-import org.apache.hadoop.mapred.OutputCollector;
-import org.apache.hadoop.mapred.Reducer;
-import org.apache.hadoop.mapred.Reporter;
-import org.apache.hadoop.mapred.TextInputFormat;
-import org.apache.hadoop.mapred.TextOutputFormat;
-
-/**
- * <p>WordCount class.</p>
- *
- * @since 0.9.0
- */
-@SuppressWarnings("deprecation")
-public class WordCount
-{
-
- public static class Map extends MapReduceBase implements Mapper<LongWritable, Text, Text, IntWritable>
- {
- private static final IntWritable one = new IntWritable(1);
- private Text word = new Text();
-
- public void map(LongWritable key, Text value, OutputCollector<Text, IntWritable> output, Reporter reporter) throws IOException
- {
- String line = value.toString();
- StringTokenizer tokenizer = new StringTokenizer(line);
- while (tokenizer.hasMoreTokens()) {
- word.set(tokenizer.nextToken());
- output.collect(word, one);
- }
- }
- }
-
- public static class Reduce extends MapReduceBase implements Reducer<Text, IntWritable, Text, IntWritable>
- {
- public void reduce(Text key, Iterator<IntWritable> values, OutputCollector<Text, IntWritable> output, Reporter reporter) throws IOException
- {
- int sum = 0;
- while (values.hasNext()) {
- sum += values.next().get();
- }
- output.collect(key, new IntWritable(sum));
- }
- }
-
- public void run(String[] args) throws Exception
- {
-
- JobConf conf = new JobConf(this.getClass());
- conf.setJobName("wordcount");
-
- conf.setOutputKeyClass(Text.class);
- conf.setOutputValueClass(IntWritable.class);
-
- conf.setMapperClass(Map.class);
- conf.setCombinerClass(Reduce.class);
- conf.setReducerClass(Reduce.class);
-
- conf.setInputFormat(TextInputFormat.class);
- conf.setOutputFormat(TextOutputFormat.class);
-
- FileInputFormat.setInputPaths(conf, new Path(args[0]));
- FileOutputFormat.setOutputPath(conf, new Path(args[1]));
-
- JobClient.runJob(conf);
- }
-
- public static void main(String[] args) throws Exception
- {
- new WordCount().run(args);
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/mroperator/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/demos/mroperator/src/main/resources/META-INF/properties.xml b/demos/mroperator/src/main/resources/META-INF/properties.xml
deleted file mode 100644
index ce95ec9..0000000
--- a/demos/mroperator/src/main/resources/META-INF/properties.xml
+++ /dev/null
@@ -1,88 +0,0 @@
-<?xml version="1.0"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<configuration>
- <!-- Log Count Demo -->
- <property>
- <name>dt.application.LogsCountDemo.operator.Mapper.dirName</name>
- <value></value>
- </property>
- <property>
- <name>dt.application.LogsCountDemo.operator.Mapper.partitionCount</name>
- <value></value>
- </property>
- <property>
- <name>dt.application.LogsCountDemo.operator.Console.filePath</name>
- <value></value>
- </property>
- <property>
- <name>dt.application.LogsCountDemo.operator.Console.outputFileName</name>
- <value></value>
- </property>
- <property>
- <name>dt.application.LogsCountDemo.operator.Reducer.attr.PARTITIONER</name>
- <value>com.datatorrent.common.partitioner.StatelessPartitioner:1</value>
- </property>
-
- <!-- Word Count Demo -->
- <property>
- <name>dt.application.WordCountDemo.operator.Mapper.dirName</name>
- <value></value>
- </property>
- <property>
- <name>dt.application.WordCountDemo.operator.Mapper.partitionCount</name>
- <value></value>
- </property>
- <property>
- <name>dt.application.WordCountDemo.operator.Console.filePath</name>
- <value></value>
- </property>
- <property>
- <name>dt.application.WordCountDemo.operator.Console.outputFileName</name>
- <value></value>
- </property>
- <property>
- <name>dt.application.WordCountDemo.operator.Reducer.attr.PARTITIONER</name>
- <value>com.datatorrent.common.partitioner.StatelessPartitioner:1</value>
- </property>
-
- <!-- Inverted Index Demo -->
- <property>
- <name>dt.application.InvertedIndexDemo.operator.Mapper.dirName</name>
- <value></value>
- </property>
- <property>
- <name>dt.application.InvertedIndexDemo.operator.Mapper.partitionCount</name>
- <value></value>
- </property>
- <property>
- <name>dt.application.InvertedIndexDemo.operator.Console.filePath</name>
- <value></value>
- </property>
- <property>
- <name>dt.application.InvertedIndexDemo.operator.Console.outputFileName</name>
- <value></value>
- </property>
- <property>
- <name>dt.application.LogsCountDemo.operator.Reducer.attr.PARTITIONER</name>
- <value>com.datatorrent.common.partitioner.StatelessPartitioner:1</value>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/mroperator/src/site/conf/my-app-conf1.xml
----------------------------------------------------------------------
diff --git a/demos/mroperator/src/site/conf/my-app-conf1.xml b/demos/mroperator/src/site/conf/my-app-conf1.xml
deleted file mode 100644
index f35873b..0000000
--- a/demos/mroperator/src/site/conf/my-app-conf1.xml
+++ /dev/null
@@ -1,27 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<configuration>
- <property>
- <name>dt.attr.MASTER_MEMORY_MB</name>
- <value>1024</value>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/mroperator/src/test/java/com/datatorrent/demos/mroperator/MapOperatorTest.java
----------------------------------------------------------------------
diff --git a/demos/mroperator/src/test/java/com/datatorrent/demos/mroperator/MapOperatorTest.java b/demos/mroperator/src/test/java/com/datatorrent/demos/mroperator/MapOperatorTest.java
deleted file mode 100644
index 0f330e8..0000000
--- a/demos/mroperator/src/test/java/com/datatorrent/demos/mroperator/MapOperatorTest.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.mroperator;
-
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-
-import org.junit.Assert;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TestWatcher;
-import org.junit.runner.Description;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.serializer.SerializationFactory;
-import org.apache.hadoop.io.serializer.Serializer;
-import org.apache.hadoop.mapred.FileInputFormat;
-import org.apache.hadoop.mapred.InputSplit;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.TextInputFormat;
-
-import com.datatorrent.lib.testbench.CollectorTestSink;
-
-@SuppressWarnings("deprecation")
-public class MapOperatorTest
-{
-
- private static Logger LOG = LoggerFactory.getLogger(MapOperatorTest.class);
-
- @Rule
- public TestMeta testMeta = new TestMeta();
- /**
- * Test node logic emits correct results
- */
- @Test
- public void testNodeProcessing() throws Exception
- {
- testNodeProcessingSchema(new MapOperator<LongWritable, Text, Text, IntWritable>());
- }
-
- public void testNodeProcessingSchema(MapOperator<LongWritable, Text, Text, IntWritable> oper) throws IOException
- {
-
- CollectorTestSink sortSink = new CollectorTestSink();
- oper.output.setSink(sortSink);
-
- oper.setMapClass(WordCount.Map.class);
- oper.setCombineClass(WordCount.Reduce.class);
- oper.setDirName(testMeta.testDir);
- oper.setConfigFile(null);
- oper.setInputFormatClass(TextInputFormat.class);
-
- Configuration conf = new Configuration();
- JobConf jobConf = new JobConf(conf);
- FileInputFormat.setInputPaths(jobConf, new Path(testMeta.testDir));
- TextInputFormat inputFormat = new TextInputFormat();
- inputFormat.configure(jobConf);
- InputSplit[] splits = inputFormat.getSplits(jobConf, 1);
- SerializationFactory serializationFactory = new SerializationFactory(conf);
- Serializer keySerializer = serializationFactory.getSerializer(splits[0].getClass());
- keySerializer.open(oper.getOutstream());
- keySerializer.serialize(splits[0]);
- oper.setInputSplitClass(splits[0].getClass());
- keySerializer.close();
- oper.setup(null);
- oper.beginWindow(0);
- oper.emitTuples();
- oper.emitTuples();
- oper.endWindow();
- oper.beginWindow(1);
- oper.emitTuples();
- oper.endWindow();
-
- Assert.assertEquals("number emitted tuples", 3, sortSink.collectedTuples.size());
- for (Object o : sortSink.collectedTuples) {
- LOG.debug(o.toString());
- }
- LOG.debug("Done testing round\n");
- oper.teardown();
- }
-
- public static class TestMeta extends TestWatcher
- {
- public final String file1 = "file1";
- public String baseDir;
- public String testDir;
-
- @Override
- protected void starting(org.junit.runner.Description description)
- {
- String methodName = description.getMethodName();
- String className = description.getClassName();
- baseDir = "target/" + className;
- testDir = baseDir + "/" + methodName;
- try {
- FileUtils.forceMkdir(new File(testDir));
- } catch (IOException ex) {
- throw new RuntimeException(ex);
- }
- createFile(testDir + "/" + file1, "1\n2\n3\n1\n2\n3\n");
- }
-
- private void createFile(String fileName, String data)
- {
- BufferedWriter output = null;
- try {
- output = new BufferedWriter(new FileWriter(new File(fileName)));
- output.write(data);
- } catch (IOException ex) {
- throw new RuntimeException(ex);
- } finally {
- if (output != null) {
- try {
- output.close();
- } catch (IOException ex) {
- LOG.error("not able to close the output stream: ", ex);
- }
- }
- }
- }
-
- @Override
- protected void finished(Description description)
- {
- try {
- FileUtils.deleteDirectory(new File(baseDir));
- } catch (IOException ex) {
- throw new RuntimeException(ex);
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/mroperator/src/test/java/com/datatorrent/demos/mroperator/ReduceOperatorTest.java
----------------------------------------------------------------------
diff --git a/demos/mroperator/src/test/java/com/datatorrent/demos/mroperator/ReduceOperatorTest.java b/demos/mroperator/src/test/java/com/datatorrent/demos/mroperator/ReduceOperatorTest.java
deleted file mode 100644
index b85f8ad..0000000
--- a/demos/mroperator/src/test/java/com/datatorrent/demos/mroperator/ReduceOperatorTest.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.mroperator;
-
-import org.junit.Assert;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.Text;
-
-import com.datatorrent.lib.testbench.CollectorTestSink;
-import com.datatorrent.lib.util.KeyHashValPair;
-
-public class ReduceOperatorTest
-{
- private static Logger logger = LoggerFactory.getLogger(ReduceOperatorTest.class);
-
- /**
- * Test node logic emits correct results
- */
- @Test
- public void testNodeProcessing() throws Exception
- {
- testNodeProcessingSchema(new ReduceOperator<Text, IntWritable,Text, IntWritable>());
- }
-
- @SuppressWarnings({ "rawtypes", "unchecked" })
- public void testNodeProcessingSchema(ReduceOperator<Text, IntWritable,Text, IntWritable> oper)
- {
- oper.setReduceClass(WordCount.Reduce.class);
- oper.setConfigFile(null);
- oper.setup(null);
-
- CollectorTestSink sortSink = new CollectorTestSink();
- oper.output.setSink(sortSink);
-
- oper.beginWindow(0);
- oper.inputCount.process(new KeyHashValPair<Integer, Integer>(1, 1));
- oper.input.process(new KeyHashValPair<Text, IntWritable>(new Text("one"), new IntWritable(1)));
- oper.input.process(new KeyHashValPair<Text, IntWritable>(new Text("one"), new IntWritable(1)));
- oper.input.process(new KeyHashValPair<Text, IntWritable>(new Text("two"), new IntWritable(1)));
- oper.endWindow();
-
- oper.beginWindow(1);
- oper.input.process(new KeyHashValPair<Text, IntWritable>(new Text("one"), new IntWritable(1)));
- oper.input.process(new KeyHashValPair<Text, IntWritable>(new Text("two"), new IntWritable(1)));
- oper.input.process(new KeyHashValPair<Text, IntWritable>(new Text("two"), new IntWritable(1)));
- oper.inputCount.process(new KeyHashValPair<Integer, Integer>(1, -1));
- oper.endWindow();
- Assert.assertEquals("number emitted tuples", 2, sortSink.collectedTuples.size());
- for (Object o : sortSink.collectedTuples) {
- logger.debug(o.toString());
- }
- logger.debug("Done testing round\n");
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/mroperator/src/test/java/com/datatorrent/demos/mroperator/WordCountMRApplicationTest.java
----------------------------------------------------------------------
diff --git a/demos/mroperator/src/test/java/com/datatorrent/demos/mroperator/WordCountMRApplicationTest.java b/demos/mroperator/src/test/java/com/datatorrent/demos/mroperator/WordCountMRApplicationTest.java
deleted file mode 100644
index bd732c1..0000000
--- a/demos/mroperator/src/test/java/com/datatorrent/demos/mroperator/WordCountMRApplicationTest.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.mroperator;
-
-import java.io.File;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-import org.junit.Assert;
-import org.junit.Rule;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-
-import com.google.common.collect.Maps;
-
-import com.datatorrent.api.LocalMode;
-
-public class WordCountMRApplicationTest
-{
- private static Logger LOG = LoggerFactory.getLogger(WordCountMRApplicationTest.class);
- @Rule
- public MapOperatorTest.TestMeta testMeta = new MapOperatorTest.TestMeta();
-
- @Test
- public void testSomeMethod() throws Exception
- {
- LocalMode lma = LocalMode.newInstance();
- Configuration conf = new Configuration(false);
- conf.set("dt.application.WordCountDemo.operator.Mapper.dirName", testMeta.testDir);
- conf.setInt("dt.application.WordCountDemo.operator.Mapper.partitionCount", 1);
- conf.set("dt.application.WordCountDemo.operator.Console.filePath", testMeta.testDir);
- conf.set("dt.application.WordCountDemo.operator.Console.outputFileName", "output.txt");
- lma.prepareDAG(new NewWordCountApplication(), conf);
- LocalMode.Controller lc = lma.getController();
- lc.setHeartbeatMonitoringEnabled(false);
- lc.run(5000);
- lc.shutdown();
- List<String> readLines = FileUtils.readLines(new File(testMeta.testDir + "/output.txt"));
- Map<String,Integer> readMap = Maps.newHashMap();
- Iterator<String> itr = readLines.iterator();
- while (itr.hasNext()) {
- String[] splits = itr.next().split("=");
- readMap.put(splits[0],Integer.valueOf(splits[1]));
- }
- Map<String,Integer> expectedMap = Maps.newHashMap();
- expectedMap.put("1",2);
- expectedMap.put("2",2);
- expectedMap.put("3",2);
- Assert.assertEquals("expected reduced data ", expectedMap, readMap);
- LOG.info("read lines {}", readLines);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/mroperator/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/demos/mroperator/src/test/resources/log4j.properties b/demos/mroperator/src/test/resources/log4j.properties
deleted file mode 100644
index cf0d19e..0000000
--- a/demos/mroperator/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,43 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-log4j.rootLogger=DEBUG,CONSOLE
-
-log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
-log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
-log4j.appender.CONSOLE.threshold=${test.log.console.threshold}
-test.log.console.threshold=DEBUG
-
-log4j.appender.RFA=org.apache.log4j.RollingFileAppender
-log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
-log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
-log4j.appender.RFA.File=/tmp/app.log
-
-# to enable, add SYSLOG to rootLogger
-log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender
-log4j.appender.SYSLOG.syslogHost=127.0.0.1
-log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout
-log4j.appender.SYSLOG.layout.conversionPattern=${dt.cid} %-5p [%t] %c{2} %x - %m%n
-log4j.appender.SYSLOG.Facility=LOCAL1
-
-log4j.logger.org=info
-#log4j.logger.org.apache.commons.beanutils=warn
-log4j.logger.com.datatorrent=debug
-log4j.logger.org.apache.apex=debug
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/pi/pom.xml
----------------------------------------------------------------------
diff --git a/demos/pi/pom.xml b/demos/pi/pom.xml
deleted file mode 100644
index 30c2dc1..0000000
--- a/demos/pi/pom.xml
+++ /dev/null
@@ -1,45 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>pi-demo</artifactId>
- <packaging>jar</packaging>
-
- <name>Apache Apex Malhar Pi Demo</name>
- <description>Apex demo applications that calculate the value of Pi. This is a starting point to understand how Apex works.</description>
-
- <parent>
- <groupId>org.apache.apex</groupId>
- <artifactId>malhar-demos</artifactId>
- <version>3.7.0-SNAPSHOT</version>
- </parent>
-
- <dependencies>
- <dependency>
- <groupId>it.unimi.dsi</groupId>
- <artifactId>fastutil</artifactId>
- <version>6.6.4</version>
- </dependency>
- </dependencies>
-
-</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/pi/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/demos/pi/src/assemble/appPackage.xml b/demos/pi/src/assemble/appPackage.xml
deleted file mode 100644
index 4138cf2..0000000
--- a/demos/pi/src/assemble/appPackage.xml
+++ /dev/null
@@ -1,59 +0,0 @@
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
- <id>appPackage</id>
- <formats>
- <format>jar</format>
- </formats>
- <includeBaseDirectory>false</includeBaseDirectory>
- <fileSets>
- <fileSet>
- <directory>${basedir}/target/</directory>
- <outputDirectory>/app</outputDirectory>
- <includes>
- <include>${project.artifactId}-${project.version}.jar</include>
- </includes>
- </fileSet>
- <fileSet>
- <directory>${basedir}/target/deps</directory>
- <outputDirectory>/lib</outputDirectory>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/site/conf</directory>
- <outputDirectory>/conf</outputDirectory>
- <includes>
- <include>*.xml</include>
- </includes>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/main/resources/META-INF</directory>
- <outputDirectory>/META-INF</outputDirectory>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/main/resources/app</directory>
- <outputDirectory>/app</outputDirectory>
- </fileSet>
- </fileSets>
-
-</assembly>
-
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/pi/src/main/java/com/datatorrent/demos/pi/Application.java
----------------------------------------------------------------------
diff --git a/demos/pi/src/main/java/com/datatorrent/demos/pi/Application.java b/demos/pi/src/main/java/com/datatorrent/demos/pi/Application.java
deleted file mode 100644
index 55ffe92..0000000
--- a/demos/pi/src/main/java/com/datatorrent/demos/pi/Application.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.pi;
-
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.DAG.Locality;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-import com.datatorrent.lib.io.ConsoleOutputOperator;
-import com.datatorrent.lib.testbench.RandomEventGenerator;
-
-/**
- * Monte Carlo PI estimation demo : <br>
- * This application computes value of PI using Monte Carlo pi estimation
- * formula.
- * <p>
- * Running Java Test or Main app in IDE:
- *
- * <pre>
- * LocalMode.runApp(new Application(), 600000); // 10 min run
- * </pre>
- *
- * Run Success : <br>
- * For successful deployment and run, user should see something like the
- * following output on the console (since the input sequence of random numbers
- * can vary from one run to the next, there will be some variation in the
- * output values):
- *
- * <pre>
- * 3.1430480549199085
- * 3.1423454157782515
- * 3.1431377245508982
- * 3.142078799249531
- * 2013-06-18 10:43:18,335 [main] INFO stram.StramLocalCluster run - Application finished.
- * </pre>
- *
- * Application DAG : <br>
- * <img src="doc-files/Application.gif" width=600px > <br>
- * <br>
- *
- * Streaming Window Size : 1000 ms(1 Sec) <br>
- * Operator Details : <br>
- * <ul>
- * <li><b>The rand Operator : </b> This operator generates random integer
- * between 0-30k. <br>
- * Class : {@link com.datatorrent.lib.testbench.RandomEventGenerator}<br>
- * StateFull : No</li>
- * <li><b>The calc operator : </b> This operator computes value of pi using
- * monte carlo estimation. <br>
- * Class : com.datatorrent.demos.pi.PiCalculateOperator <br>
- * StateFull : No</li>
- * <li><b>The operator Console: </b> This operator just outputs the input tuples
- * to the console (or stdout). You can use other output adapters if needed.<br>
- * </li>
- * </ul>
- *
- * @since 0.3.2
- */
-@ApplicationAnnotation(name = "PiDemo")
-public class Application implements StreamingApplication
-{
- private final Locality locality = null;
-
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- RandomEventGenerator rand = dag.addOperator("rand", new RandomEventGenerator());
- PiCalculateOperator calc = dag.addOperator("picalc", new PiCalculateOperator());
- ConsoleOutputOperator console = dag.addOperator("console", new ConsoleOutputOperator());
- dag.addStream("rand_calc", rand.integer_data, calc.input).setLocality(locality);
- dag.addStream("rand_console",calc.output, console.input).setLocality(locality);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/pi/src/main/java/com/datatorrent/demos/pi/ApplicationAppData.java
----------------------------------------------------------------------
diff --git a/demos/pi/src/main/java/com/datatorrent/demos/pi/ApplicationAppData.java b/demos/pi/src/main/java/com/datatorrent/demos/pi/ApplicationAppData.java
deleted file mode 100644
index 328bb10..0000000
--- a/demos/pi/src/main/java/com/datatorrent/demos/pi/ApplicationAppData.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.pi;
-
-import java.net.URI;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.DAG.Locality;
-import com.datatorrent.api.Operator;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-import com.datatorrent.lib.appdata.schemas.SchemaUtils;
-import com.datatorrent.lib.appdata.snapshot.AppDataSnapshotServerMap;
-import com.datatorrent.lib.io.ConsoleOutputOperator;
-import com.datatorrent.lib.io.PubSubWebSocketAppDataQuery;
-import com.datatorrent.lib.io.PubSubWebSocketAppDataResult;
-import com.datatorrent.lib.testbench.RandomEventGenerator;
-
-/**
- * Monte Carlo PI estimation demo : <br>
- * This application computes value of PI using Monte Carlo pi estimation
- * formula.
- * <p>
- * Very similar to PiDemo but data is also written to an App Data operator for visualization.
- * <p>
- * Running Java Test or Main app in IDE:
- *
- * <pre>
- * LocalMode.runApp(new Application(), 600000); // 10 min run
- * </pre>
- *
- * Run Success : <br>
- * For successful deployment and run, user should see something like the
- * following output on the console (since the input sequence of random numbers
- * can vary from one run to the next, there will be some variation in the
- * output values):
- *
- * <pre>
- * 3.1430480549199085
- * 3.1423454157782515
- * 3.1431377245508982
- * 3.142078799249531
- * 2013-06-18 10:43:18,335 [main] INFO stram.StramLocalCluster run - Application finished.
- * </pre>
- *
- * Application DAG : <br>
- * <img src="doc-files/Application.gif" width=600px > <br>
- * <br>
- *
- * Streaming Window Size : 1000 ms(1 Sec) <br>
- * Operator Details : <br>
- * <ul>
- * <li><b>The rand Operator : </b> This operator generates random integer
- * between 0-30k. <br>
- * Class : {@link com.datatorrent.lib.testbench.RandomEventGenerator}<br>
- * StateFull : No</li>
- * <li><b>The calc operator : </b> This operator computes value of pi using
- * monte carlo estimation. <br>
- * Class : com.datatorrent.demos.pi.PiCalculateOperator <br>
- * StateFull : No</li>
- * <li><b>The operator Console: </b> This operator just outputs the input tuples
- * to the console (or stdout). You can use other output adapters if needed.<br>
- * </li>
- * </ul>
- *
- * @since 0.3.2
- */
-@ApplicationAnnotation(name = "PiDemoAppData")
-public class ApplicationAppData implements StreamingApplication
-{
- public static final String SNAPSHOT_SCHEMA = "PiDemoDataSchema.json";
-
- private final Locality locality = null;
-
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- RandomEventGenerator rand = dag.addOperator("rand", new RandomEventGenerator());
- PiCalculateOperator calc = dag.addOperator("picalc", new PiCalculateOperator());
-
-
- dag.addStream("rand_calc", rand.integer_data, calc.input).setLocality(locality);
-
- String gatewayAddress = dag.getValue(DAG.GATEWAY_CONNECT_ADDRESS);
-
- if (StringUtils.isEmpty(gatewayAddress)) {
- throw new RuntimeException("Error: No GATEWAY_CONNECT_ADDRESS");
- }
-
- URI uri = URI.create("ws://" + gatewayAddress + "/pubsub");
-
- AppDataSnapshotServerMap snapshotServer = dag.addOperator("SnapshotServer", new AppDataSnapshotServerMap());
-
- String snapshotServerJSON = SchemaUtils.jarResourceFileToString(SNAPSHOT_SCHEMA);
-
- snapshotServer.setSnapshotSchemaJSON(snapshotServerJSON);
-
- PubSubWebSocketAppDataQuery wsQuery = new PubSubWebSocketAppDataQuery();
- wsQuery.enableEmbeddedMode();
- snapshotServer.setEmbeddableQueryInfoProvider(wsQuery);
-
- PubSubWebSocketAppDataResult wsResult = dag.addOperator("QueryResult", new PubSubWebSocketAppDataResult());
-
- wsQuery.setUri(uri);
- wsResult.setUri(uri);
- Operator.InputPort<String> queryResultPort = wsResult.input;
-
- NamedValueList<Object> adaptor = dag.addOperator("adaptor", new NamedValueList<Object>());
- ConsoleOutputOperator console = dag.addOperator("console", new ConsoleOutputOperator());
-
- dag.addStream("PiValues", calc.output, adaptor.inPort, console.input).setLocality(locality);;
- dag.addStream("NamedPiValues", adaptor.outPort, snapshotServer.input);
- dag.addStream("Result", snapshotServer.queryResult, queryResultPort);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/pi/src/main/java/com/datatorrent/demos/pi/ApplicationWithScript.java
----------------------------------------------------------------------
diff --git a/demos/pi/src/main/java/com/datatorrent/demos/pi/ApplicationWithScript.java b/demos/pi/src/main/java/com/datatorrent/demos/pi/ApplicationWithScript.java
deleted file mode 100644
index 0796608..0000000
--- a/demos/pi/src/main/java/com/datatorrent/demos/pi/ApplicationWithScript.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.pi;
-
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-import com.datatorrent.lib.io.ConsoleOutputOperator;
-import com.datatorrent.lib.script.JavaScriptOperator;
-import com.datatorrent.lib.stream.RoundRobinHashMap;
-import com.datatorrent.lib.testbench.RandomEventGenerator;
-
-/**
- * Monte Carlo PI estimation demo : <br>
- * This application computes value of PI using Monte Carlo pi estimation
- * formula. This demo inputs formula using java script operator.
- *
- * Running Java Test or Main app in IDE:
- *
- * <pre>
- * LocalMode.runApp(new Application(), 600000); // 10 min run
- * </pre>
- *
- * Run Success : <br>
- * For successful deployment and run, user should see following output on
- * console:
- * <pre>
- * 2013-06-25 11:44:25,842 [container-2] DEBUG stram.StramChildAgent updateOperatorStatus - container-2 pendingDeploy []
- * 2013-06-25 11:44:25,929 [ServerHelper-1-1] DEBUG netlet.AbstractClient send - allocating new sendBuffer4Offers of size 16384 for Server.Subscriber{type=rrhm_calc/3.inBindings, mask=0, partitions=null}
- * 3.16
- * 3.15
- * 3.1616
- * 3.148
- * 3.1393846153846154
- * </pre>
- *
- * * Application DAG : <br>
- * <img src="doc-files/ApplicationScript.gif" width=600px > <br>
- * <br>
- *
- * Streaming Window Size : 1000 ms(1 Sec) <br>
- * Operator Details : <br>
- * <ul>
- * <li><b>The rand Operator : </b> This operator generates random integer
- * between 0-30k. <br>
- * Class : {@link com.datatorrent.lib.testbench.RandomEventGenerator} <br>
- * StateFull : No</li>
- * <li><b>The rrhm Operator : </b> This operator takes input from random generator
- * creates tuples of (x,y) in round robin fashion. <br>
- * Class : {@link com.datatorrent.lib.stream.RandomEventGenerator} <br>
- * StateFull : Yes, tuple is emitted after (x, y) values have been aggregated.</li>
- * <li><b>The calc operator : </b> This is java script operator implementing <br>
- * Class : {@link com.datatorrent.lib.math.Script} <br>
- * StateFull : No</li>
- * <li><b>The operator Console: </b> This operator just outputs the input tuples
- * to the console (or stdout). User can use any output adapter. <br>
- * .</li>
- * </ul>
- *
- * @since 0.3.2
- */
-@ApplicationAnnotation(name = "PiJavaScriptDemo")
-public class ApplicationWithScript implements StreamingApplication
-{
-
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- int maxValue = 30000;
-
- RandomEventGenerator rand = dag.addOperator("rand", new RandomEventGenerator());
- rand.setMinvalue(0);
- rand.setMaxvalue(maxValue);
-
- RoundRobinHashMap<String,Object> rrhm = dag.addOperator("rrhm", new RoundRobinHashMap<String, Object>());
- rrhm.setKeys(new String[]{"x", "y"});
-
- JavaScriptOperator calc = dag.addOperator("picalc", new JavaScriptOperator());
- calc.setPassThru(false);
- calc.put("i",0);
- calc.put("count",0);
- calc.addSetupScript("function pi() { if (x*x+y*y <= " + maxValue * maxValue + ") { i++; } count++; return i / count * 4; }");
-
- calc.setInvoke("pi");
-
- dag.addStream("rand_rrhm", rand.integer_data, rrhm.data);
- dag.addStream("rrhm_calc", rrhm.map, calc.inBindings);
-
- ConsoleOutputOperator console = dag.addOperator("console", new ConsoleOutputOperator());
- dag.addStream("rand_console",calc.result, console.input);
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/pi/src/main/java/com/datatorrent/demos/pi/Calculator.java
----------------------------------------------------------------------
diff --git a/demos/pi/src/main/java/com/datatorrent/demos/pi/Calculator.java b/demos/pi/src/main/java/com/datatorrent/demos/pi/Calculator.java
deleted file mode 100644
index 221ecc0..0000000
--- a/demos/pi/src/main/java/com/datatorrent/demos/pi/Calculator.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.pi;
-
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-import com.datatorrent.lib.io.ConsoleOutputOperator;
-import com.datatorrent.lib.math.Division;
-import com.datatorrent.lib.math.LogicalCompareToConstant;
-import com.datatorrent.lib.math.MultiplyByConstant;
-import com.datatorrent.lib.math.RunningAverage;
-import com.datatorrent.lib.math.Sigma;
-import com.datatorrent.lib.math.SquareCalculus;
-import com.datatorrent.lib.stream.AbstractAggregator;
-import com.datatorrent.lib.stream.ArrayListAggregator;
-import com.datatorrent.lib.stream.Counter;
-import com.datatorrent.lib.testbench.RandomEventGenerator;
-
-/**
- * <p>Calculator class.</p>
- *
- * @since 0.3.2
- */
-@ApplicationAnnotation(name = "PiLibraryDemo")
-public class Calculator implements StreamingApplication
-{
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- /* keep generating random values between 0 and 30000 */
- RandomEventGenerator xyGenerator = dag.addOperator("GenerateX", RandomEventGenerator.class);
-
- /* calculate square of each of the values it receives */
- SquareCalculus squareOperator = dag.addOperator("SquareX", SquareCalculus.class);
-
- /* pair the consecutive values */
- AbstractAggregator<Integer> pairOperator = dag.addOperator("PairXY", new ArrayListAggregator<Integer>());
- Sigma<Integer> sumOperator = dag.addOperator("SumXY", new Sigma<Integer>());
- LogicalCompareToConstant<Integer> comparator = dag.addOperator("AnalyzeLocation", new LogicalCompareToConstant<Integer>());
- comparator.setConstant(30000 * 30000);
- Counter inCircle = dag.addOperator("CountInCircle", Counter.class);
- Counter inSquare = dag.addOperator("CountInSquare", Counter.class);
- Division division = dag.addOperator("Ratio", Division.class);
- MultiplyByConstant multiplication = dag.addOperator("InstantPI", MultiplyByConstant.class);
- multiplication.setMultiplier(4);
- RunningAverage average = dag.addOperator("AveragePI", new RunningAverage());
- ConsoleOutputOperator oper = dag.addOperator("Console", new ConsoleOutputOperator());
-
- dag.addStream("x", xyGenerator.integer_data, squareOperator.input);
- dag.addStream("sqr", squareOperator.integerResult, pairOperator.input);
- dag.addStream("x2andy2", pairOperator.output, sumOperator.input);
- dag.addStream("x2plusy2", sumOperator.integerResult, comparator.input, inSquare.input);
- dag.addStream("inCirclePoints", comparator.greaterThan, inCircle.input);
- dag.addStream("numerator", inCircle.output, division.numerator);
- dag.addStream("denominator", inSquare.output, division.denominator);
- dag.addStream("ratio", division.doubleQuotient, multiplication.input);
- dag.addStream("instantPi", multiplication.doubleProduct, average.input);
- dag.addStream("averagePi", average.doubleAverage, oper.input);
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/pi/src/main/java/com/datatorrent/demos/pi/NamedValueList.java
----------------------------------------------------------------------
diff --git a/demos/pi/src/main/java/com/datatorrent/demos/pi/NamedValueList.java b/demos/pi/src/main/java/com/datatorrent/demos/pi/NamedValueList.java
deleted file mode 100644
index c50e17e..0000000
--- a/demos/pi/src/main/java/com/datatorrent/demos/pi/NamedValueList.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.pi;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import javax.validation.constraints.NotNull;
-
-import com.datatorrent.api.Context.OperatorContext;
-import com.datatorrent.api.DefaultInputPort;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.common.util.BaseOperator;
-
-/**
- * <p>An operator which converts a raw value to a named value singleton list.</p>
- * AppDataSnapshotServerMap.input accepts a List<Map<String,Object>> so we use this operator to
- * convert individual values to a singleton list of a named value
- * <p>
- * @displayNamed Value
- * @tags count
- * @since 3.2.0
- */
-public class NamedValueList<T> extends BaseOperator
-{
- @NotNull
- private String valueName;
-
- private List<Map<String, T>> valueList;
- private Map<String, T> valueMap;
-
- public final transient DefaultInputPort<T> inPort = new DefaultInputPort<T>()
- {
- @Override
- public void process(T val)
- {
- valueMap.put(valueName, val);
- outPort.emit(valueList);
- }
- };
-
- public final transient DefaultOutputPort<List<Map<String, T>>> outPort = new DefaultOutputPort<>();
-
- @Override
- public void setup(OperatorContext context)
- {
- valueMap = new HashMap<>();
- valueMap.put(valueName, null);
- valueList = Collections.singletonList(valueMap);
- }
-
- @Override
- public void teardown()
- {
- }
-
- @Override
- public void beginWindow(long windowId)
- {
- }
-
- @Override
- public void endWindow()
- {
- }
-
- public String getValueName()
- {
- return valueName;
- }
-
- public void setValueName(String name)
- {
- valueName = name;
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/pi/src/main/java/com/datatorrent/demos/pi/PiCalculateOperator.java
----------------------------------------------------------------------
diff --git a/demos/pi/src/main/java/com/datatorrent/demos/pi/PiCalculateOperator.java b/demos/pi/src/main/java/com/datatorrent/demos/pi/PiCalculateOperator.java
deleted file mode 100644
index 8e61991..0000000
--- a/demos/pi/src/main/java/com/datatorrent/demos/pi/PiCalculateOperator.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.pi;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.datatorrent.api.Context.OperatorContext;
-import com.datatorrent.api.DefaultInputPort;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.common.util.BaseOperator;
-
-/**
- * This operator implements Monte Carlo estimation of pi. For points randomly distributed points on
- * square circle. pi ~= Number of points in circle/Total number of points * 4.
- *
- * @since 0.3.2
- */
-public class PiCalculateOperator extends BaseOperator
-{
- private transient int x = -1;
- private transient int y = -1;
- private int base;
- private long inArea = 0;
- private long totalArea = 0;
- public final transient DefaultInputPort<Integer> input = new DefaultInputPort<Integer>()
- {
- @Override
- public void process(Integer tuple)
- {
- if (x == -1) {
- x = tuple;
- } else {
- y = tuple;
- if (x * x + y * y <= base) {
- inArea++;
- }
- totalArea++;
- x = y = -1;
- }
- }
-
- };
- public final transient DefaultOutputPort<Double> output = new DefaultOutputPort<Double>();
-
- @Override
- public void setup(OperatorContext context)
- {
- logger.info("inArea {} totalArea {}", inArea, totalArea);
- }
-
- @Override
- public void beginWindow(long windowId)
- {
- }
-
- @Override
- public void endWindow()
- {
- output.emit((double)inArea / totalArea * 4);
- }
-
- public void setBase(int num)
- {
- base = num;
- }
-
- public int getBase()
- {
- return base;
- }
-
- private static Logger logger = LoggerFactory.getLogger(PiCalculateOperator.class);
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/pi/src/main/java/com/datatorrent/demos/pi/doc-files/Application.gif
----------------------------------------------------------------------
diff --git a/demos/pi/src/main/java/com/datatorrent/demos/pi/doc-files/Application.gif b/demos/pi/src/main/java/com/datatorrent/demos/pi/doc-files/Application.gif
deleted file mode 100644
index 9545c6c..0000000
Binary files a/demos/pi/src/main/java/com/datatorrent/demos/pi/doc-files/Application.gif and /dev/null differ
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/pi/src/main/java/com/datatorrent/demos/pi/doc-files/ApplicationScript.gif
----------------------------------------------------------------------
diff --git a/demos/pi/src/main/java/com/datatorrent/demos/pi/doc-files/ApplicationScript.gif b/demos/pi/src/main/java/com/datatorrent/demos/pi/doc-files/ApplicationScript.gif
deleted file mode 100644
index 29f9ef8..0000000
Binary files a/demos/pi/src/main/java/com/datatorrent/demos/pi/doc-files/ApplicationScript.gif and /dev/null differ
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/pi/src/main/java/com/datatorrent/demos/pi/package-info.java
----------------------------------------------------------------------
diff --git a/demos/pi/src/main/java/com/datatorrent/demos/pi/package-info.java b/demos/pi/src/main/java/com/datatorrent/demos/pi/package-info.java
deleted file mode 100644
index 11614d6..0000000
--- a/demos/pi/src/main/java/com/datatorrent/demos/pi/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/**
- * Pi calculation demonstration application.
- */
-package com.datatorrent.demos.pi;
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/pi/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/demos/pi/src/main/resources/META-INF/properties.xml b/demos/pi/src/main/resources/META-INF/properties.xml
deleted file mode 100644
index 96d097d..0000000
--- a/demos/pi/src/main/resources/META-INF/properties.xml
+++ /dev/null
@@ -1,109 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<configuration>
- <!-- Memory settings for all demos -->
- <property>
- <name>dt.attr.MASTER_MEMORY_MB</name>
- <value>1024</value>
- </property>
- <property>
- <name>dt.application.*.operator.*.attr.MEMORY_MB</name>
- <value>256</value>
- </property>
- <property>
- <name>dt.application.*.operator.*.attr.JVM_OPTIONS</name>
- <value>-Xmx128M</value>
- </property>
- <property>
- <name>dt.application.*.operator.*.port.*.attr.BUFFER_MEMORY_MB</name>
- <value>128</value>
- </property>
-
- <!-- PiDemo -->
- <property>
- <name>dt.application.PiDemo.operator.rand.minvalue</name>
- <value>0</value>
- </property>
- <property>
- <name>dt.application.PiDemo.operator.rand.maxvalue</name>
- <value>30000</value>
- </property>
- <property>
- <name>dt.application.PiDemo.operator.picalc.base</name>
- <value>900000000</value>
- </property>
- <property>
- <name>dt.application.PiDemo.operator.adaptor.valueName</name>
- <value>piValue</value>
- </property>
-
- <!-- PiDemoAppData -->
- <property>
- <name>dt.application.PiDemoAppData.operator.rand.minvalue</name>
- <value>0</value>
- </property>
- <property>
- <name>dt.application.PiDemoAppData.operator.rand.maxvalue</name>
- <value>30000</value>
- </property>
- <property>
- <name>dt.application.PiDemoAppData.operator.picalc.base</name>
- <value>900000000</value>
- </property>
- <property>
- <name>dt.application.PiDemoAppData.operator.adaptor.valueName</name>
- <value>piValue</value>
- </property>
- <property>
- <name>dt.application.PiDemoAppData.operator.Query.topic</name>
- <value>PiDemoQuery</value>
- </property>
- <property>
- <name>dt.application.PiDemoAppData.operator.QueryResult.topic</name>
- <value>PiDemoQueryResult</value>
- </property>
- <property>
- <name>dt.application.PiDemoAppData.operator.SnapshotServer.embeddableQueryInfoProvider.topic</name>
- <value>PiDemoQuery</value>
- </property>
-
- <!-- PiLibraryDemo -->
- <property>
- <name>dt.application.PiLibraryDemo.operator.GenerateX.minvalue</name>
- <value>0</value>
- </property>
- <property>
- <name>dt.application.PiLibraryDemo.operator.GenerateX.maxvalue</name>
- <value>30000</value>
- </property>
- <property>
- <name>dt.application.PiLibraryDemo.operator.PairXY.size</name>
- <value>2</value>
- </property>
-
- <!-- PiJavaScriptDemo -->
- <property>
- <name>dt.application.PiJavaScriptDemo.operator.rand.tuplesBlast</name>
- <value>9</value>
- </property>
-
-</configuration>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/pi/src/main/resources/PiDemoDataSchema.json
----------------------------------------------------------------------
diff --git a/demos/pi/src/main/resources/PiDemoDataSchema.json b/demos/pi/src/main/resources/PiDemoDataSchema.json
deleted file mode 100644
index 47db8eb..0000000
--- a/demos/pi/src/main/resources/PiDemoDataSchema.json
+++ /dev/null
@@ -1,3 +0,0 @@
-{
- "values": [{"name": "piValue", "type": "double"}]
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/pi/src/main/resources/app/PiJsonDemo.json
----------------------------------------------------------------------
diff --git a/demos/pi/src/main/resources/app/PiJsonDemo.json b/demos/pi/src/main/resources/app/PiJsonDemo.json
deleted file mode 100644
index c077213..0000000
--- a/demos/pi/src/main/resources/app/PiJsonDemo.json
+++ /dev/null
@@ -1,52 +0,0 @@
-{
- "description": "Pi JSON Demo that is intended to demonstrate the capability of specifying an app using JSON",
- "operators": [
- {
- "name": "rand",
- "class": "com.datatorrent.lib.testbench.RandomEventGenerator",
- "properties": {
- "minvalue": 0,
- "maxvalue": 30000
- }
- },
- {
- "name": "picalc",
- "class": "com.datatorrent.demos.pi.PiCalculateOperator",
- "properties": {
- "base": 900000000
- }
- },
- {
- "name": "console",
- "class": "com.datatorrent.lib.io.ConsoleOutputOperator"
- }
- ],
- "streams": [
- {
- "name": "rand_calc",
- "source": {
- "operatorName": "rand",
- "portName": "integer_data"
- },
- "sinks": [
- {
- "operatorName": "picalc",
- "portName": "input"
- }
- ]
- },
- {
- "name": "calc_console",
- "source": {
- "operatorName": "picalc",
- "portName": "output"
- },
- "sinks": [
- {
- "operatorName": "console",
- "portName": "input"
- }
- ]
- }
- ]
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/pi/src/test/java/com/datatorrent/demos/pi/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/demos/pi/src/test/java/com/datatorrent/demos/pi/ApplicationTest.java b/demos/pi/src/test/java/com/datatorrent/demos/pi/ApplicationTest.java
deleted file mode 100644
index d8881c2..0000000
--- a/demos/pi/src/test/java/com/datatorrent/demos/pi/ApplicationTest.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.pi;
-
-import org.junit.Test;
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.LocalMode;
-
-/**
- *
- */
-public class ApplicationTest
-{
- @Test
- public void testSomeMethod() throws Exception
- {
- LocalMode lma = LocalMode.newInstance();
- Configuration conf = new Configuration(false);
- conf.addResource("dt-site-pi.xml");
- lma.prepareDAG(new Application(), conf);
- LocalMode.Controller lc = lma.getController();
- lc.run(10000);
-
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/pi/src/test/java/com/datatorrent/demos/pi/ApplicationWithScriptTest.java
----------------------------------------------------------------------
diff --git a/demos/pi/src/test/java/com/datatorrent/demos/pi/ApplicationWithScriptTest.java b/demos/pi/src/test/java/com/datatorrent/demos/pi/ApplicationWithScriptTest.java
deleted file mode 100644
index 8f35e5a..0000000
--- a/demos/pi/src/test/java/com/datatorrent/demos/pi/ApplicationWithScriptTest.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.pi;
-
-import org.junit.Test;
-
-import com.datatorrent.api.LocalMode;
-
-/**
- *
- */
-public class ApplicationWithScriptTest
-{
- @Test
- public void testSomeMethod() throws Exception
- {
- LocalMode.runApp(new ApplicationWithScript(), 10000);
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/pi/src/test/java/com/datatorrent/demos/pi/CalculatorTest.java
----------------------------------------------------------------------
diff --git a/demos/pi/src/test/java/com/datatorrent/demos/pi/CalculatorTest.java b/demos/pi/src/test/java/com/datatorrent/demos/pi/CalculatorTest.java
deleted file mode 100644
index 21079d7..0000000
--- a/demos/pi/src/test/java/com/datatorrent/demos/pi/CalculatorTest.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.pi;
-
-import org.junit.Test;
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.LocalMode;
-
-/**
- *
- */
-public class CalculatorTest
-{
- @Test
- public void testSomeMethod() throws Exception
- {
- LocalMode lma = LocalMode.newInstance();
- Configuration conf = new Configuration(false);
- conf.addResource("dt-site-pilibrary.xml");
- lma.prepareDAG(new Calculator(), conf);
- LocalMode.Controller lc = lma.getController();
- lc.run(10000);
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/pi/src/test/resources/dt-site-pi.xml
----------------------------------------------------------------------
diff --git a/demos/pi/src/test/resources/dt-site-pi.xml b/demos/pi/src/test/resources/dt-site-pi.xml
deleted file mode 100644
index 6230cdd..0000000
--- a/demos/pi/src/test/resources/dt-site-pi.xml
+++ /dev/null
@@ -1,40 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<configuration>
- <property>
- <name>dt.application.PiDemo.class</name>
- <value>com.datatorrent.demos.pi.Application</value>
- <description>An alias for the application</description>
-</property>
-<property>
- <name>dt.application.PiDemo.operator.rand.minvalue</name>
- <value>0</value>
-</property>
-<property>
- <name>dt.application.PiDemo.operator.rand.maxvalue</name>
- <value>30000</value>
-</property>
-<property>
- <name>dt.application.PiDemo.operator.picalc.base</name>
- <value>900000000</value>
-</property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/pi/src/test/resources/dt-site-pilibrary.xml
----------------------------------------------------------------------
diff --git a/demos/pi/src/test/resources/dt-site-pilibrary.xml b/demos/pi/src/test/resources/dt-site-pilibrary.xml
deleted file mode 100644
index 01c7ef3..0000000
--- a/demos/pi/src/test/resources/dt-site-pilibrary.xml
+++ /dev/null
@@ -1,45 +0,0 @@
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<configuration>
- <property>
- <name>dt.application.PiLibraryDemo.class</name>
- <value>com.datatorrent.demos.pi.Calculator</value>
- <description>An alias for the application</description>
-</property>
-<property>
- <name>dt.application.PiLibraryDemo.operator.GenerateX.minvalue</name>
- <value>0</value>
-</property>
-<property>
- <name>dt.application.PiLibraryDemo.operator.GenerateX.maxvalue</name>
- <value>30000</value>
-</property>
-<property>
- <name>dt.application.PiLibraryDemo.operator.PairXY.size</name>
- <value>2</value>
-</property>
-<!--
-<property>
- <name>dt.application.PiLibraryDemo.operator.AnalyzeLocation.constant</name>
- <value>900000000</value>
-</property> -->
-
-</configuration>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/pi/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/demos/pi/src/test/resources/log4j.properties b/demos/pi/src/test/resources/log4j.properties
deleted file mode 100644
index cf0d19e..0000000
--- a/demos/pi/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,43 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-log4j.rootLogger=DEBUG,CONSOLE
-
-log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
-log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
-log4j.appender.CONSOLE.threshold=${test.log.console.threshold}
-test.log.console.threshold=DEBUG
-
-log4j.appender.RFA=org.apache.log4j.RollingFileAppender
-log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
-log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
-log4j.appender.RFA.File=/tmp/app.log
-
-# to enable, add SYSLOG to rootLogger
-log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender
-log4j.appender.SYSLOG.syslogHost=127.0.0.1
-log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout
-log4j.appender.SYSLOG.layout.conversionPattern=${dt.cid} %-5p [%t] %c{2} %x - %m%n
-log4j.appender.SYSLOG.Facility=LOCAL1
-
-log4j.logger.org=info
-#log4j.logger.org.apache.commons.beanutils=warn
-log4j.logger.com.datatorrent=debug
-log4j.logger.org.apache.apex=debug
[28/30] apex-malhar git commit: Renamed demos to examples. Packages
and artifactid names are changed as suggested.
Posted by th...@apache.org.
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/CreditCardAmountSamplerOperator.java
----------------------------------------------------------------------
diff --git a/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/CreditCardAmountSamplerOperator.java b/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/CreditCardAmountSamplerOperator.java
deleted file mode 100644
index a232fd4..0000000
--- a/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/CreditCardAmountSamplerOperator.java
+++ /dev/null
@@ -1,206 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.frauddetect;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-import org.codehaus.jackson.JsonFactory;
-import org.codehaus.jackson.map.ObjectMapper;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.commons.lang.mutable.MutableLong;
-
-import com.datatorrent.api.DefaultInputPort;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.common.util.BaseOperator;
-import com.datatorrent.demos.frauddetect.util.JsonUtils;
-import com.datatorrent.lib.util.KeyValPair;
-
-/**
- * An operator to alert in case a transaction of a small lowAmount is followed by a transaction which is significantly larger for a given credit card number.
- * This is done for each transaction. This also means that this happens for each individual credit card.
- * It accepts merchant transaction object and for each CC listed in the transaction(s), checks for the transaction amounts. An alert is raised if the transaction
- * lowAmount is significantly > the lowest amt in this window.
- *
- * @since 0.9.0
- */
-public class CreditCardAmountSamplerOperator extends BaseOperator
-{
- private final transient JsonFactory jsonFactory = new JsonFactory();
- private final transient ObjectMapper mapper = new ObjectMapper(jsonFactory);
- private static final Logger logger = LoggerFactory.getLogger(Application.class);
- // Factor to be applied to existing lowAmount to flag potential alerts.
- private double threshold = 9500;
- private Map<String, CreditCardInfo> ccTxnMap = new HashMap<String, CreditCardInfo>();
- //private Map<String, MutableLong> ccQueryTxnMap = new HashMap<String, MutableLong>();
- private List<CreditCardAlertData> alerts = new ArrayList<CreditCardAlertData>();
- //private List<CreditCardAlertData> userAlerts = new ArrayList<CreditCardAlertData>();
- private static final String ALERT_MSG =
- "Potential fraudulent CC transactions (small one USD %d followed by large USD %d) performed using credit card: %s";
- public final transient DefaultOutputPort<String> ccAlertOutputPort = new DefaultOutputPort<String>();
- /*
- public final transient DefaultOutputPort<Map<String, Object>> ccUserAlertOutputPort = new DefaultOutputPort<Map<String, Object>>();
- */
- public final transient DefaultOutputPort<Map<String, Object>> ccAlertNotificationPort = new DefaultOutputPort<Map<String, Object>>();
-
- public double getThreshold()
- {
- return threshold;
- }
-
- public void setThreshold(double threshold)
- {
- this.threshold = threshold;
- }
-
- private void processTuple(KeyValPair<MerchantKey, CreditCardData> tuple, Map<String, CreditCardInfo> txMap)
- {
- String fullCcNum = tuple.getValue().fullCcNum;
- long ccAmount = tuple.getValue().amount;
- MerchantKey key = tuple.getKey();
-
- CreditCardInfo cardInfo = txMap.get(fullCcNum);
-
- if (cardInfo != null) {
- long currentSmallValue = cardInfo.lowAmount.longValue();
- if (ccAmount < currentSmallValue) {
- cardInfo.lowAmount.setValue(ccAmount);
- cardInfo.time = key.time;
- } else if (ccAmount > (currentSmallValue + threshold)) {
- // If the transaction lowAmount is > 70% of the min. lowAmount, send an alert.
-
- CreditCardAlertData data = new CreditCardAlertData();
-
- data.merchantId = key.merchantId;
- data.terminalId = key.terminalId == null ? 0 : key.terminalId;
- data.zipCode = key.zipCode;
- data.merchantType = key.merchantType;
- data.fullCcNum = fullCcNum;
- data.small = currentSmallValue;
- data.large = ccAmount;
- data.threshold = threshold;
- data.userGenerated = key.userGenerated;
- data.time = System.currentTimeMillis();
-
- alerts.add(data);
-
- /*
- if (userGenerated){
- userAlerts.add(data);
- }
- */
- ccAlertNotificationPort.emit(getOutputData(data));
-
- // Any high value transaction after a low value transaction with difference greater than threshold
- // will trigger the alert. Not resetting the low value also helps in a system generated transaction
- // alert not resetting the low value from a user generated transaction
- //txMap.remove(fullCcNum);
- }
- } else {
- cardInfo = new CreditCardInfo();
- cardInfo.lowAmount.setValue(ccAmount);
- cardInfo.time = key.time;
- txMap.put(fullCcNum, cardInfo);
- }
- }
-
- public transient DefaultInputPort<KeyValPair<MerchantKey, CreditCardData>> inputPort =
- new DefaultInputPort<KeyValPair<MerchantKey, CreditCardData>>()
- {
- //
- // This function checks if a CC entry exists.
- // If so, it checks whether the current transaction is for an lowAmount lesser than the one stored in the hashmap. If so, this becomes the min. transaction lowAmount.
- // If the lowAmount is > 70% of the existing lowAmount in the hash map, raise an alert.
- //
- @Override
- public void process(KeyValPair<MerchantKey, CreditCardData> tuple)
- {
-
- processTuple(tuple, ccTxnMap);
-
- }
-
- };
-
- @Override
- public void endWindow()
- {
-
- for (CreditCardAlertData data : alerts) {
- try {
- ccAlertOutputPort.emit(JsonUtils.toJson(data));
- } catch (IOException e) {
- logger.warn("Exception while converting object to JSON", e);
- }
- }
-
- //for (CreditCardAlertData data: userAlerts) {
- /*for (CreditCardAlertData data: alerts) {
- ccAlertNotificationPort.emit(getOutputData(data));
- }*/
-
- long ctime = System.currentTimeMillis();
- Iterator<Map.Entry<String, CreditCardInfo>> iterator = ccTxnMap.entrySet().iterator();
- while (iterator.hasNext()) {
- Map.Entry<String, CreditCardInfo> entry = iterator.next();
- long time = entry.getValue().time;
- if ((ctime - time) > 60000) {
- iterator.remove();
- }
- }
-
- //ccTxnMap.clear();
- alerts.clear();
-
- //ccQueryTxnMap.clear();
- //userAlerts.clear();
- }
-
- private static class CreditCardInfo
- {
- MutableLong lowAmount = new MutableLong();
- Long time;
- }
-
- private Map<String, Object> getOutputData(CreditCardAlertData data)
- {
- Map<String, Object> output = new HashMap<String, Object>();
- output.put("message", String.format(ALERT_MSG, data.small, data.large, data.fullCcNum));
- output.put("alertType", "smallThenLarge");
- output.put("userGenerated", "" + data.userGenerated);
- output.put("alertData", data);
-
- try {
- String str = mapper.writeValueAsString(output);
- logger.debug("Alert generated: " + str + " userGenerated: " + data.userGenerated);
- } catch (Exception exc) {
- //ignore
- }
-
- return output;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/CreditCardData.java
----------------------------------------------------------------------
diff --git a/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/CreditCardData.java b/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/CreditCardData.java
deleted file mode 100644
index 68a2cf7..0000000
--- a/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/CreditCardData.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.frauddetect;
-
-/**
- * Credit Card Data
- *
- * @since 0.9.0
- */
-public class CreditCardData
-{
- public String fullCcNum;
- public long amount;
-
- public CreditCardData()
- {
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/MerchantKey.java
----------------------------------------------------------------------
diff --git a/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/MerchantKey.java b/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/MerchantKey.java
deleted file mode 100644
index a6204c9..0000000
--- a/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/MerchantKey.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.frauddetect;
-
-
-import java.io.Serializable;
-
-/**
- * A time-based key for merchant data.
- *
- * @since 0.9.0
- */
-public class MerchantKey implements Serializable
-{
- public String merchantId;
- public Integer terminalId;
- public Integer zipCode;
- public String country;
- public MerchantTransaction.MerchantType merchantType;
- public Long time;
- public boolean userGenerated;
-
- public MerchantKey()
- {
- }
-
- @Override
- public int hashCode()
- {
- int key = 0;
- if (merchantId != null) {
- key |= (1 << 1);
- key |= (merchantId.hashCode());
- }
- if (terminalId != null) {
- key |= (1 << 2);
- key |= (terminalId << 2);
- }
- if (zipCode != null) {
- key |= (1 << 3);
- key |= (zipCode << 3);
- }
- if (country != null) {
- key |= (1 << 4);
- key |= (country.hashCode());
- }
- if (merchantType != null) {
- key |= (1 << 5);
- key |= (merchantType.hashCode());
- }
- return key;
- }
-
- @Override
- public boolean equals(Object obj)
- {
- if (!(obj instanceof MerchantKey)) {
- return false;
- }
- MerchantKey mkey = (MerchantKey)obj;
- return checkStringEqual(this.merchantId, mkey.merchantId)
- && checkIntEqual(this.terminalId, mkey.terminalId)
- && checkIntEqual(this.zipCode, mkey.zipCode)
- && checkStringEqual(this.country, mkey.country)
- && checkIntEqual(this.merchantType.ordinal(), mkey.merchantType.ordinal());
- }
-
- private boolean checkIntEqual(Integer a, Integer b)
- {
- if ((a == null) && (b == null)) {
- return true;
- }
- if ((a != null) && (b != null) && a.intValue() == b.intValue()) {
- return true;
- }
- return false;
- }
-
- private boolean checkStringEqual(String a, String b)
- {
- if ((a == null) && (b == null)) {
- return true;
- }
- if ((a != null) && a.equals(b)) {
- return true;
- }
- return false;
- }
-
- @Override
- public String toString()
- {
- StringBuilder sb = new StringBuilder();
- if (merchantId != null) {
- sb.append("|1:").append(merchantId);
- }
- if (terminalId != null) {
- sb.append("|2:").append(terminalId);
- }
- if (zipCode != null) {
- sb.append("|3:").append(zipCode);
- }
- if (country != null) {
- sb.append("|4:").append(country);
- }
- if (merchantType != null) {
- sb.append("|5:").append(merchantType);
- }
- return sb.toString();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/MerchantQueryInputHandler.java
----------------------------------------------------------------------
diff --git a/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/MerchantQueryInputHandler.java b/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/MerchantQueryInputHandler.java
deleted file mode 100644
index c57f86d..0000000
--- a/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/MerchantQueryInputHandler.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.frauddetect;
-
-import java.util.Map;
-
-/**
- * Common utility class that can be used by all other operators to handle user input
- * captured from the Web socket input port.
- *
- * @since 0.9.0
- */
-public class MerchantQueryInputHandler
-{
- public static final String KEY_DATA = "data";
- public static final String KEY_MERCHANT_ID = "merchantId";
- public static final String KEY_TERMINAL_ID = "terminalId";
- public static final String KEY_ZIP_CODE = "zipCode";
-
- public static MerchantKey process(Map<String, Object> tuple)
- {
- String merchantId = null;
- Integer terminalId = null;
- Integer zipCode = null;
-
- // ignoring other top-level attributes.
- Map<String, Object> data = (Map<String, Object>)tuple.get(KEY_DATA);
- if (data.get(KEY_MERCHANT_ID) != null) {
- merchantId = (String)data.get(KEY_MERCHANT_ID);
- }
- if (data.get(KEY_TERMINAL_ID) != null) {
- terminalId = (Integer)data.get(KEY_TERMINAL_ID);
- }
- if (data.get(KEY_ZIP_CODE) != null) {
- zipCode = (Integer)data.get(KEY_ZIP_CODE);
- }
-
- MerchantKey key = new MerchantKey();
- key.merchantId = merchantId;
- key.terminalId = terminalId;
- key.zipCode = zipCode;
- key.country = "USA";
- if (merchantId != null) {
- key.merchantType = key.merchantId.equalsIgnoreCase(MerchantTransactionGenerator.merchantIds[2])
- || key.merchantId.equalsIgnoreCase(MerchantTransactionGenerator.merchantIds[3])
- ? MerchantTransaction.MerchantType.INTERNET
- : MerchantTransaction.MerchantType.BRICK_AND_MORTAR;
- }
- return key;
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/MerchantTransaction.java
----------------------------------------------------------------------
diff --git a/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/MerchantTransaction.java b/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/MerchantTransaction.java
deleted file mode 100644
index 75e279c..0000000
--- a/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/MerchantTransaction.java
+++ /dev/null
@@ -1,202 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.frauddetect;
-
-import java.io.Serializable;
-
-/**
- * POJO for BIN Alert related data.
- *
- * @since 0.9.0
- */
-public class MerchantTransaction implements Serializable
-{
- public enum MerchantType
- {
- UNDEFINED, BRICK_AND_MORTAR, INTERNET
- }
-
- public enum TransactionType
- {
- UNDEFINED, POS
- }
-
- public String ccNum;
- public String bankIdNum;
- public String fullCcNum;
- public Long amount;
- public String merchantId;
- public Integer terminalId;
- public Integer zipCode;
- public String country;
- public MerchantType merchantType = MerchantType.UNDEFINED;
- public TransactionType transactionType = TransactionType.UNDEFINED;
- public Long time;
- public boolean userGenerated;
-
- public MerchantTransaction()
- {
- }
-
- @Override
- public int hashCode()
- {
- int key = 0;
- if (ccNum != null) {
- key |= (1 << 1);
- key |= (ccNum.hashCode());
- }
- if (bankIdNum != null) {
- key |= (1 << 2);
- key |= (bankIdNum.hashCode());
- }
- if (amount != null) {
- key |= (1 << 6);
- key |= (amount << 4);
- }
- if (merchantId != null) {
- key |= (1 << 3);
- key |= (merchantId.hashCode());
- }
- if (terminalId != null) {
- key |= (1 << 4);
- key |= (terminalId << 2);
- }
- if (zipCode != null) {
- key |= (1 << 5);
- key |= (zipCode << 3);
- }
- if (country != null) {
- key |= (1 << 7);
- key |= (country.hashCode());
- }
- if (merchantType != null) {
- key |= (1 << 8);
- key |= (merchantType.hashCode());
- }
- if (transactionType != null) {
- key |= (1 << 9);
- key |= (transactionType.hashCode());
- }
- if (fullCcNum != null) {
- key |= (1 << 10);
- key |= (fullCcNum.hashCode());
- }
- if (time != null) {
- key |= (1 << 11);
- key |= (time << 2);
- }
-
- return key;
- }
-
- @Override
- public boolean equals(Object obj)
- {
- if (!(obj instanceof MerchantTransaction)) {
- return false;
- }
- MerchantTransaction mtx = (MerchantTransaction)obj;
- return checkStringEqual(this.ccNum, mtx.ccNum)
- && checkStringEqual(this.bankIdNum, mtx.bankIdNum)
- && checkLongEqual(this.amount, mtx.amount)
- && checkStringEqual(this.merchantId, mtx.merchantId)
- && checkIntEqual(this.terminalId, mtx.terminalId)
- && checkIntEqual(this.zipCode, mtx.zipCode)
- && checkStringEqual(this.country, mtx.country)
- && checkIntEqual(this.merchantType.ordinal(), mtx.merchantType.ordinal())
- && checkIntEqual(this.transactionType.ordinal(), mtx.transactionType.ordinal())
- && checkStringEqual(this.fullCcNum, mtx.fullCcNum)
- && checkLongEqual(this.time, mtx.time);
- }
-
- private boolean checkIntEqual(Integer a, Integer b)
- {
- if ((a == null) && (b == null)) {
- return true;
- }
- if ((a != null) && (b != null) && a.intValue() == b.intValue()) {
- return true;
- }
- return false;
- }
-
- private boolean checkLongEqual(Long a, Long b)
- {
- if ((a == null) && (b == null)) {
- return true;
- }
- if ((a != null) && (b != null) && a.longValue() == b.longValue()) {
- return true;
- }
- return false;
- }
-
- private boolean checkStringEqual(String a, String b)
- {
- if ((a == null) && (b == null)) {
- return true;
- }
- if ((a != null) && a.equals(b)) {
- return true;
- }
- return false;
- }
-
- @Override
- public String toString()
- {
- StringBuilder sb = new StringBuilder();
- if (ccNum != null) {
- sb.append("|0:").append(ccNum);
- }
- if (bankIdNum != null) {
- sb.append("|1:").append(bankIdNum);
- }
- if (fullCcNum != null) {
- sb.append("|2:").append(fullCcNum);
- }
- if (amount != null) {
- sb.append("|3:").append(amount);
- }
- if (merchantId != null) {
- sb.append("|4:").append(merchantId);
- }
- if (terminalId != null) {
- sb.append("|5:").append(terminalId);
- }
- if (zipCode != null) {
- sb.append("|6:").append(zipCode);
- }
- if (country != null) {
- sb.append("|7:").append(country);
- }
- if (merchantType != null) {
- sb.append("|8:").append(merchantType);
- }
- if (transactionType != null) {
- sb.append("|9:").append(transactionType);
- }
- if (time != null) {
- sb.append("|10:").append(time);
- }
- return sb.toString();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/MerchantTransactionBucketOperator.java
----------------------------------------------------------------------
diff --git a/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/MerchantTransactionBucketOperator.java b/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/MerchantTransactionBucketOperator.java
deleted file mode 100644
index 243137d..0000000
--- a/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/MerchantTransactionBucketOperator.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.frauddetect;
-
-import java.text.DecimalFormat;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.commons.lang.mutable.MutableDouble;
-import org.apache.commons.lang.mutable.MutableLong;
-
-import com.datatorrent.api.DefaultInputPort;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.common.util.BaseOperator;
-import com.datatorrent.lib.util.KeyValPair;
-
-/**
- * A bucket-like operator to accept merchant transaction object and dissipate the
- * transaction amount to the further downstream operator for calculating min, max and std-deviation.
- *
- * @since 0.9.0
- */
-public class MerchantTransactionBucketOperator extends BaseOperator
-{
- private static final Logger logger = LoggerFactory.getLogger(MerchantTransactionBucketOperator.class);
- /*
- public final transient DefaultOutputPort<KeyValPair<MerchantKey, String>> binOutputPort =
- new DefaultOutputPort<KeyValPair<MerchantKey, String>>();
- */
- public final transient DefaultOutputPort<KeyValPair<KeyValPair<MerchantKey, String>, Integer>> binCountOutputPort =
- new DefaultOutputPort<KeyValPair<KeyValPair<MerchantKey, String>, Integer>>();
- public final transient DefaultOutputPort<KeyValPair<MerchantKey, Long>> txOutputPort =
- new DefaultOutputPort<KeyValPair<MerchantKey, Long>>();
- public final transient DefaultOutputPort<KeyValPair<MerchantKey, CreditCardData>> ccAlertOutputPort =
- new DefaultOutputPort<KeyValPair<MerchantKey, CreditCardData>>();
- public final transient DefaultOutputPort<Map<String, Object>> summaryTxnOutputPort =
- new DefaultOutputPort<Map<String, Object>>();
- private MutableLong totalTxns = new MutableLong(0);
- private MutableLong txnsInLastSecond = new MutableLong(0);
- private MutableDouble amtInLastSecond = new MutableDouble(0);
- private transient DecimalFormat amtFormatter = new DecimalFormat("#.##");
- public transient DefaultInputPort<MerchantTransaction> inputPort = new DefaultInputPort<MerchantTransaction>()
- {
- @Override
- public void process(MerchantTransaction tuple)
- {
- processTuple(tuple);
- }
-
- };
- public transient DefaultInputPort<MerchantTransaction> txUserInputPort = new DefaultInputPort<MerchantTransaction>()
- {
- @Override
- public void process(MerchantTransaction tuple)
- {
- processTuple(tuple);
- }
-
- };
-
- public void endWindow()
- {
- Map<String, Object> summary = new HashMap<String, Object>();
- double avg;
- if (txnsInLastSecond.longValue() == 0) {
- avg = 0;
- } else {
- avg = amtInLastSecond.doubleValue() / txnsInLastSecond.longValue();
- }
- summary.put("totalTxns", totalTxns);
- summary.put("txnsInLastSecond", txnsInLastSecond);
- summary.put("amtInLastSecond", amtFormatter.format(amtInLastSecond));
- summary.put("avgAmtInLastSecond", amtFormatter.format(avg));
- summaryTxnOutputPort.emit(summary);
- txnsInLastSecond.setValue(0);
- amtInLastSecond.setValue(0);
- }
-
- private void processTuple(MerchantTransaction tuple)
- {
- emitBankIdNumTuple(tuple, binCountOutputPort);
- emitMerchantKeyTuple(tuple, txOutputPort);
- emitCreditCardKeyTuple(tuple, ccAlertOutputPort);
- totalTxns.increment();
- txnsInLastSecond.increment();
- amtInLastSecond.add(tuple.amount);
- }
-
- private void emitMerchantKeyTuple(MerchantTransaction tuple, DefaultOutputPort<KeyValPair<MerchantKey, Long>> outputPort)
- {
- MerchantKey key = getMerchantKey(tuple);
- KeyValPair<MerchantKey, Long> keyValPair = new KeyValPair<MerchantKey, Long>(key, tuple.amount);
- outputPort.emit(keyValPair);
- }
-
- //private void emitBankIdNumTuple(MerchantTransaction tuple, DefaultOutputPort<KeyValPair<MerchantKey, String>> outputPort)
- private void emitBankIdNumTuple(MerchantTransaction tuple, DefaultOutputPort<KeyValPair<KeyValPair<MerchantKey, String>, Integer>> outputPort)
- {
- MerchantKey key = getMerchantKey(tuple);
- KeyValPair<MerchantKey, String> keyValPair = new KeyValPair<MerchantKey, String>(key, tuple.bankIdNum);
- outputPort.emit(new KeyValPair<KeyValPair<MerchantKey, String>, Integer>(keyValPair, 1));
- }
-
- private void emitCreditCardKeyTuple(MerchantTransaction tuple, DefaultOutputPort<KeyValPair<MerchantKey, CreditCardData>> outputPort)
- {
- MerchantKey key = getMerchantKey(tuple);
-
- CreditCardData data = new CreditCardData();
- data.fullCcNum = tuple.fullCcNum;
- data.amount = tuple.amount;
-
- KeyValPair<MerchantKey, CreditCardData> keyValPair = new KeyValPair<MerchantKey, CreditCardData>(key, data);
- outputPort.emit(keyValPair);
- }
-
- private MerchantKey getMerchantKey(MerchantTransaction tuple)
- {
- MerchantKey key = new MerchantKey();
- key.merchantId = tuple.merchantId;
- key.terminalId = tuple.terminalId;
- key.zipCode = tuple.zipCode;
- key.country = tuple.country;
- key.merchantType = tuple.merchantType;
- key.userGenerated = tuple.userGenerated;
- key.time = tuple.time;
- return key;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/MerchantTransactionGenerator.java
----------------------------------------------------------------------
diff --git a/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/MerchantTransactionGenerator.java b/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/MerchantTransactionGenerator.java
deleted file mode 100644
index 49b61aa..0000000
--- a/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/MerchantTransactionGenerator.java
+++ /dev/null
@@ -1,210 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.frauddetect;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.datatorrent.api.Context;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.api.InputOperator;
-
-import com.datatorrent.common.util.BaseOperator;
-import com.datatorrent.demos.frauddetect.util.JsonUtils;
-
-/**
- * Information tuple generator with randomness.
- *
- * @since 0.9.0
- */
-public class MerchantTransactionGenerator extends BaseOperator implements InputOperator
-{
- private final Random randomNum = new Random();
- public static final int[] zipCodes = {94086, 94087, 94088, 94089, 94090, 94091, 94092, 94093};
- public static final String[] merchantIds = {"Wal-Mart", "Target", "Amazon", "Apple", "Sears", "Macys", "JCPenny", "Levis"};
-// public static final String bankIdNums[] = { "1111 1111 1111", "2222 2222 2222", "3333 3333 3333", "4444 4444 4444", "5555 5555 5555", "6666 6666 6666", "7777 7777 7777", "8888 8888 8888"};
-// public static final String ccNums[] = { "0001", "0002", "0003", "0004", "0005", "0006", "0007", "0008"};
-// public static final String bankIdNums[] = { "1111 1111 1111", "2222 2222 2222", "3333 3333 3333", "4444 4444 4444"};
-// public static final String ccNums[] = { "0001", "0002", "0003", "0004"};
-// public static final int zipCodes[] = { 94086, 94087, 94088, 94089, 94090};
-// public static final String merchantIds[] = { "Wal-Mart", "Target", "Amazon", "Apple"};
-// private int bankIdNumMin = 0;
-// private int bankIdNumMax = bankIdNums.length - 1;
-// private int ccMin = 0;
-// private int ccMax = ccNums.length - 1;
- private int amountMin = 1;
- private int amountMax = 400;
- private int merchantIdMin = 0;
- private int merchantIdMax = merchantIds.length - 1;
- private int terminalIdMin = 1;
- private int terminalIdMax = 8;
- private int zipMin = 0;
- private int zipMax = zipCodes.length - 1;
- private int tupleBlastSize = 2000;
- private boolean stopGeneration = false;
-
- @Override
- public void setup(Context.OperatorContext context)
- {
- super.setup(context);
- }
-
- public transient DefaultOutputPort<MerchantTransaction> txOutputPort =
- new DefaultOutputPort<MerchantTransaction>();
- public transient DefaultOutputPort<String> txDataOutputPort =
- new DefaultOutputPort<String>();
-
- @Override
- public void emitTuples()
- {
- int count = 0;
- List<MerchantTransaction> txList = new ArrayList();
-
- while (!stopGeneration && count < getTupleBlastSize()) {
-
- String bankIdNum = genBankIdNum();
- String ccNum = genCC();
- int merchant = genMerchantId();
- int terminal = genTerminalId();
- int zip = genZip();
-
- long amount = genAmount();
-
-// int bankIdNum = 1;
-// int ccNum = 2;
-// long amount = 5000;
-// int merchant = 3;
-// int terminal = 4;
-// int zip = 0;
-
- MerchantTransaction tx = new MerchantTransaction();
- tx.bankIdNum = bankIdNum;
- tx.ccNum = ccNum;
- tx.fullCcNum = tx.bankIdNum + " " + tx.ccNum;
- tx.amount = amount;
- tx.merchantId = merchantIds[merchant];
-
- // its INTERNET merchant
- tx.merchantType = merchant == 2 || merchant == 3
- ? MerchantTransaction.MerchantType.INTERNET
- : MerchantTransaction.MerchantType.BRICK_AND_MORTAR;
-
- tx.transactionType = MerchantTransaction.TransactionType.POS;
-
- // set terminal only for a BRICK_AND_MORTAR merchant
- if (merchant != 2 && merchant != 3) {
- tx.terminalId = terminal;
- }
- tx.zipCode = zipCodes[zip];
- tx.country = "USA";
- tx.time = System.currentTimeMillis();
-
- tx.userGenerated = false;
-
- txOutputPort.emit(tx);
-
- txList.add(tx);
-
- count++;
- }
- for (MerchantTransaction txData : txList) {
- try {
- txDataOutputPort.emit(JsonUtils.toJson(txData));
- } catch (IOException e) {
- logger.warn("Exception while converting object to JSON", e);
- }
- }
- txList.clear();
-
- try {
- Thread.sleep(100);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
-
- }
-
- public String genBankIdNum()
- {
- // Bank ID will be between 1000 0000 and 3500 0000 (25 BINs)
- int base = randomNum.nextInt(100) + 100;
- return base + "0 0000";
- }
-
- public String genCC()
- {
- // CC will be 1000 0000 to 1400 0000 (400,000 cards per BIN)
- int base = (randomNum.nextInt(100000) + 10000000);
- String baseString = Integer.toString(base);
- return baseString.substring(0, 4) + " " + baseString.substring(4);
- }
-
- public int genAmount()
- {
- int lowRange = 50;
- int range = amountMax - amountMin + randomNum.nextInt(lowRange);
- return amountMin + randomNum.nextInt(range);
- }
-
- public int genMerchantId()
- {
- int range = merchantIdMax - merchantIdMin + 1;
- return merchantIdMin + randomNum.nextInt(range);
- }
-
- public int genTerminalId()
- {
- int range = terminalIdMax - terminalIdMin + 1;
- return terminalIdMin + randomNum.nextInt(range);
- }
-
- public int genZip()
- {
- int range = zipMax - zipMin + 1;
- return zipMin + randomNum.nextInt(range);
- }
-
- public void setStopGeneration(boolean stopGeneration)
- {
- this.stopGeneration = stopGeneration;
- }
-
- /**
- * @return the tupleBlastSize
- */
- public int getTupleBlastSize()
- {
- return tupleBlastSize;
- }
-
- /**
- * @param tupleBlastSize the tupleBlastSize to set
- */
- public void setTupleBlastSize(int tupleBlastSize)
- {
- this.tupleBlastSize = tupleBlastSize;
- }
-
- private static final Logger logger = LoggerFactory.getLogger(MerchantTransactionGenerator.class);
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/MerchantTransactionInputHandler.java
----------------------------------------------------------------------
diff --git a/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/MerchantTransactionInputHandler.java b/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/MerchantTransactionInputHandler.java
deleted file mode 100644
index cdc829d..0000000
--- a/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/MerchantTransactionInputHandler.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.frauddetect;
-
-import java.util.Map;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.datatorrent.api.DefaultInputPort;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.common.util.BaseOperator;
-
-/**
- * Common utility class that can be used by all other operators to handle user input
- * captured from the Web socket input port.
- *
- * @since 0.9.0
- */
-public class MerchantTransactionInputHandler extends BaseOperator
-{
- public static final String KEY_BANK_ID_NUMBER = "bankIdNum"; // first 12 digits
- public static final String KEY_CREDIT_CARD_NUMBER = "ccNum"; // last 4 digits
- public static final String KEY_MERCHANT_ID = "merchantId";
- public static final String KEY_TERMINAL_ID = "terminalId";
- public static final String KEY_ZIP_CODE = "zipCode";
- public static final String KEY_AMOUNT = "amount";
- public transient DefaultOutputPort<MerchantTransaction> txOutputPort =
- new DefaultOutputPort<MerchantTransaction>();
- public transient DefaultInputPort<Map<String, String>> userTxInputPort = new DefaultInputPort<Map<String, String>>()
- {
- @Override
- public void process(Map<String, String> tuple)
- {
- try {
- txOutputPort.emit(processInput(tuple));
- } catch (Exception exc) {
- logger.error("Exception while handling the input", exc);
- }
- }
-
- };
-
- public MerchantTransaction processInput(Map<String, String> tuple)
- {
- String bankIdNum = null;
- String ccNum = null;
- String merchantId = null;
- Integer terminalId = null;
- Integer zipCode = null;
- Long amount = null;
- for (Map.Entry<String, String> e : tuple.entrySet()) {
- if (e.getKey().equals(KEY_BANK_ID_NUMBER)) {
- bankIdNum = e.getValue();
- }
- if (e.getKey().equals(KEY_CREDIT_CARD_NUMBER)) {
- ccNum = e.getValue();
- }
- if (e.getKey().equals(KEY_MERCHANT_ID)) {
- merchantId = e.getValue();
- }
- if (e.getKey().equals(KEY_TERMINAL_ID)) {
- terminalId = new Integer(e.getValue());
- }
- if (e.getKey().equals(KEY_ZIP_CODE)) {
- zipCode = new Integer(e.getValue());
- }
- if (e.getKey().equals(KEY_AMOUNT)) {
- amount = new Long(e.getValue());
- }
- }
-
- if (bankIdNum == null || ccNum == null || merchantId == null || terminalId == null || zipCode == null || amount == null) {
- throw new IllegalArgumentException("Missing required input!");
- }
-
- MerchantTransaction tx = new MerchantTransaction();
- tx.bankIdNum = bankIdNum;
- tx.ccNum = ccNum;
- tx.fullCcNum = bankIdNum + " " + ccNum;
- tx.merchantId = merchantId;
- tx.terminalId = terminalId;
- tx.zipCode = zipCode;
- tx.country = "USA";
- tx.amount = amount;
- tx.merchantType = tx.merchantId.equalsIgnoreCase(MerchantTransactionGenerator.merchantIds[2])
- || tx.merchantId.equalsIgnoreCase(MerchantTransactionGenerator.merchantIds[3])
- ? MerchantTransaction.MerchantType.INTERNET
- : MerchantTransaction.MerchantType.BRICK_AND_MORTAR;
- tx.transactionType = MerchantTransaction.TransactionType.POS;
-
- tx.userGenerated = true;
- tx.time = System.currentTimeMillis();
-
- return tx;
-
- }
-
- private static final Logger logger = LoggerFactory.getLogger(MerchantTransactionInputHandler.class);
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/SlidingWindowSumKeyVal.java
----------------------------------------------------------------------
diff --git a/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/SlidingWindowSumKeyVal.java b/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/SlidingWindowSumKeyVal.java
deleted file mode 100644
index 2701c14..0000000
--- a/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/SlidingWindowSumKeyVal.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.frauddetect;
-
-import java.util.ArrayList;
-
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.api.annotation.OutputPortFieldAnnotation;
-
-import com.datatorrent.lib.multiwindow.AbstractSlidingWindowKeyVal;
-import com.datatorrent.lib.util.KeyValPair;
-
-
-/**
- * Sliding window sum operator
- *
- * @since 0.9.0
- */
-public class SlidingWindowSumKeyVal<K, V extends Number> extends AbstractSlidingWindowKeyVal<K, V, SlidingWindowSumObject>
-{
-
- /**
- * Output port to emit simple moving average (SMA) of last N window as Double.
- */
- @OutputPortFieldAnnotation(optional = true)
- public final transient DefaultOutputPort<KeyValPair<K, Double>> doubleSum = new DefaultOutputPort<KeyValPair<K, Double>>();
- /**
- * Output port to emit simple moving average (SMA) of last N window as Float.
- */
- @OutputPortFieldAnnotation(optional = true)
- public final transient DefaultOutputPort<KeyValPair<K, Float>> floatSum = new DefaultOutputPort<KeyValPair<K, Float>>();
- /**
- * Output port to emit simple moving average (SMA) of last N window as Long.
- */
- @OutputPortFieldAnnotation(optional = true)
- public final transient DefaultOutputPort<KeyValPair<K, Long>> longSum = new DefaultOutputPort<KeyValPair<K, Long>>();
- /**
- * Output port to emit simple moving average (SMA) of last N window as
- * Integer.
- */
- @OutputPortFieldAnnotation(optional = true)
- public final transient DefaultOutputPort<KeyValPair<K, Integer>> integerSum = new DefaultOutputPort<KeyValPair<K, Integer>>();
-
-
- @Override
- public void processDataTuple(KeyValPair<K, V> tuple)
- {
- K key = tuple.getKey();
- ArrayList<SlidingWindowSumObject> stateList = buffer.get(key);
- if (stateList == null) {
- stateList = new ArrayList<SlidingWindowSumObject>();
- for (int i = 0; i < windowSize; ++i) {
- stateList.add(new SlidingWindowSumObject());
- }
- buffer.put(key, stateList);
- }
- SlidingWindowSumObject state = stateList.get(currentstate);
- state.add(tuple.getValue());
- }
-
- @Override
- public void emitTuple(K key, ArrayList<SlidingWindowSumObject> obj)
- {
- double sum = 0;
- for (int i = 0; i < obj.size(); ++i) {
- SlidingWindowSumObject state = obj.get(i);
- sum += state.getSum();
- }
- if (doubleSum.isConnected()) {
- doubleSum.emit(new KeyValPair<K, Double>(key, sum));
- }
- if (floatSum.isConnected()) {
- floatSum.emit(new KeyValPair<K, Float>(key, (float)sum));
- }
- if (longSum.isConnected()) {
- longSum.emit(new KeyValPair<K, Long>(key, (long)sum));
- }
- if (integerSum.isConnected()) {
- integerSum.emit(new KeyValPair<K, Integer>(key, (int)sum));
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/SlidingWindowSumObject.java
----------------------------------------------------------------------
diff --git a/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/SlidingWindowSumObject.java b/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/SlidingWindowSumObject.java
deleted file mode 100644
index 3fefb66..0000000
--- a/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/SlidingWindowSumObject.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.frauddetect;
-
-
-import org.apache.commons.lang.mutable.MutableDouble;
-
-import com.datatorrent.lib.multiwindow.SimpleMovingAverageObject;
-
-/**
- * State object for sliding window sum
- *
- * @since 0.9.0
- */
-public class SlidingWindowSumObject extends SimpleMovingAverageObject
-{
-
- MutableDouble sum = new MutableDouble(0);
-
- public void add(Number n)
- {
- sum.add(n);
- }
-
- @Override
- public double getSum()
- {
- return sum.doubleValue();
- }
-
- @Override
- public void clear()
- {
- sum.setValue(0);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/TransactionStatsAggregator.java
----------------------------------------------------------------------
diff --git a/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/TransactionStatsAggregator.java b/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/TransactionStatsAggregator.java
deleted file mode 100644
index e226af0..0000000
--- a/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/TransactionStatsAggregator.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.frauddetect;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.datatorrent.api.DefaultInputPort;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.common.util.BaseOperator;
-import com.datatorrent.demos.frauddetect.util.JsonUtils;
-import com.datatorrent.lib.util.HighLow;
-import com.datatorrent.lib.util.KeyValPair;
-
-/**
- * Operator to aggregate the min, max, sma, std-dev and variance for the given key.
- *
- * @since 0.9.0
- */
-public class TransactionStatsAggregator extends BaseOperator
-{
- public Map<MerchantKey, TransactionStatsData> aggrgateMap =
- new HashMap<MerchantKey, TransactionStatsData>();
- public final transient DefaultOutputPort<String> txDataOutputPort = new DefaultOutputPort<String>();
- public final transient DefaultInputPort<KeyValPair<MerchantKey, HighLow<Long>>> rangeInputPort =
- new DefaultInputPort<KeyValPair<MerchantKey, HighLow<Long>>>()
- {
- @Override
- public void process(KeyValPair<MerchantKey, HighLow<Long>> tuple)
- {
- TransactionStatsData data = getDataObjectFromMap(tuple.getKey());
- // HighLow is not currently typed, casting till it is fixed
- data.min = tuple.getValue().getLow();
- data.max = tuple.getValue().getHigh();
- }
-
- };
- public final transient DefaultInputPort<KeyValPair<MerchantKey, Long>> smaInputPort =
- new DefaultInputPort<KeyValPair<MerchantKey, Long>>()
- {
- @Override
- public void process(KeyValPair<MerchantKey, Long> tuple)
- {
- TransactionStatsData data = getDataObjectFromMap(tuple.getKey());
- data.sma = tuple.getValue();
- }
-
- };
-
- private TransactionStatsData getDataObjectFromMap(MerchantKey key)
- {
- TransactionStatsData data = aggrgateMap.get(key);
- if (data == null) {
- data = new TransactionStatsData();
- data.time = System.currentTimeMillis();
- data.merchantId = key.merchantId;
- data.terminalId = key.terminalId == null ? 0 : key.terminalId;
- data.zipCode = key.zipCode;
- data.merchantType = key.merchantType;
- aggrgateMap.put(key, data);
- }
- return data;
- }
-
- @Override
- public void endWindow()
- {
- for (Map.Entry<MerchantKey, TransactionStatsData> entry : aggrgateMap.entrySet()) {
- try {
- txDataOutputPort.emit(JsonUtils.toJson(entry.getValue()));
- } catch (IOException e) {
- logger.warn("Exception while converting object to JSON", e);
- }
- }
- aggrgateMap.clear();
- }
-
- private static final Logger logger = LoggerFactory.getLogger(TransactionStatsAggregator.class);
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/TransactionStatsData.java
----------------------------------------------------------------------
diff --git a/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/TransactionStatsData.java b/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/TransactionStatsData.java
deleted file mode 100644
index 4f899bc..0000000
--- a/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/TransactionStatsData.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.frauddetect;
-
-/**
- * POJO to capture transaction data related to min, max, sma, std-dev, variance.
- *
- * @since 0.9.0
- */
-public class TransactionStatsData
-{
- public String merchantId;
- public int terminalId;
- public int zipCode;
- public MerchantTransaction.MerchantType merchantType;
- public long min;
- public long max;
- public double sma;
- public long time;
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/operator/HdfsStringOutputOperator.java
----------------------------------------------------------------------
diff --git a/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/operator/HdfsStringOutputOperator.java b/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/operator/HdfsStringOutputOperator.java
deleted file mode 100644
index 4b8f851..0000000
--- a/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/operator/HdfsStringOutputOperator.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.frauddetect.operator;
-
-import java.io.File;
-
-import com.datatorrent.api.Context.DAGContext;
-import com.datatorrent.api.Context.OperatorContext;
-import com.datatorrent.lib.io.fs.AbstractFileOutputOperator;
-
-/**
- * Adapter for writing Strings to HDFS
- * <p>
- * Serializes tuples into a HDFS file.<br/>
- * </p>
- *
- * @since 0.9.4
- */
-public class HdfsStringOutputOperator extends AbstractFileOutputOperator<String>
-{
- private transient String outputFileName;
- private transient String contextId;
- private int index = 0;
-
- public HdfsStringOutputOperator()
- {
- setMaxLength(1024 * 1024);
- }
-
- @Override
- public void setup(OperatorContext context)
- {
- contextId = context.getValue(DAGContext.APPLICATION_NAME);
- outputFileName = File.separator + contextId +
- File.separator + "transactions.out.part";
- super.setup(context);
- }
-
- @Override
- public byte[] getBytesForTuple(String t)
- {
- return t.getBytes();
- }
-
- @Override
- protected String getFileName(String tuple)
- {
- return outputFileName;
- }
-
- @Override
- public String getPartFileName(String fileName, int part)
- {
- return fileName + part;
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/operator/MongoDBOutputOperator.java
----------------------------------------------------------------------
diff --git a/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/operator/MongoDBOutputOperator.java b/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/operator/MongoDBOutputOperator.java
deleted file mode 100644
index 0171c00..0000000
--- a/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/operator/MongoDBOutputOperator.java
+++ /dev/null
@@ -1,188 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.frauddetect.operator;
-
-import java.net.UnknownHostException;
-import java.util.ArrayList;
-import java.util.List;
-
-import javax.validation.constraints.NotNull;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.mongodb.DB;
-import com.mongodb.DBCollection;
-import com.mongodb.DBObject;
-import com.mongodb.MongoClient;
-import com.mongodb.WriteConcern;
-import com.mongodb.WriteResult;
-import com.mongodb.util.JSON;
-
-import com.datatorrent.api.Context;
-import com.datatorrent.api.DefaultInputPort;
-import com.datatorrent.common.util.BaseOperator;
-
-
-/**
- * Operator to write data into MongoDB
- *
- * @since 0.9.0
- */
-public class MongoDBOutputOperator extends BaseOperator
-{
- @NotNull
- protected String hostName;
- @NotNull
- protected String dataBase;
- @NotNull
- protected String collection;
-
- protected WriteConcern writeConcern = WriteConcern.ACKNOWLEDGED;
-
- protected String userName;
- protected String passWord;
-
- protected transient MongoClient mongoClient;
- protected transient DB db;
- protected transient DBCollection dbCollection;
-
- protected List<DBObject> dataList = new ArrayList<DBObject>();
-
- public MongoDBOutputOperator()
- {
- }
-
- /**
- * Take the JSON formatted string and convert it to DBObject
- */
- public final transient DefaultInputPort<String> inputPort = new DefaultInputPort<String>()
- {
- @Override
- public void process(String tuple)
- {
- dataList.add((DBObject)JSON.parse(tuple));
- }
- };
-
- @Override
- public void setup(Context.OperatorContext context)
- {
- super.setup(context);
- try {
- mongoClient = new MongoClient(hostName);
- db = mongoClient.getDB(dataBase);
- if (userName != null && passWord != null) {
- if (!db.authenticate(userName, passWord.toCharArray())) {
- throw new IllegalArgumentException("MongoDB authentication failed. Illegal username and password for MongoDB!!");
- }
- }
- dbCollection = db.getCollection(collection);
- } catch (UnknownHostException ex) {
- logger.debug(ex.toString());
- }
- }
-
- @Override
- public void beginWindow(long windowId)
- {
- // nothing
- }
-
- @Override
- public void endWindow()
- {
- logger.debug("mongo datalist size: " + dataList.size());
- if (dataList.size() > 0) {
- WriteResult result = dbCollection.insert(dataList, writeConcern);
- logger.debug("Result for MongoDB insert: " + result);
- dataList.clear();
- }
- }
-
- @Override
- public void teardown()
- {
- if (mongoClient != null) {
- mongoClient.close();
- }
- }
-
- public String getHostName()
- {
- return hostName;
- }
-
- public void setHostName(String hostName)
- {
- this.hostName = hostName;
- }
-
- public String getDataBase()
- {
- return dataBase;
- }
-
- public void setDataBase(String dataBase)
- {
- this.dataBase = dataBase;
- }
-
- public String getCollection()
- {
- return collection;
- }
-
- public void setCollection(String collection)
- {
- this.collection = collection;
- }
-
- public String getUserName()
- {
- return userName;
- }
-
- public void setUserName(String userName)
- {
- this.userName = userName;
- }
-
- public String getPassWord()
- {
- return passWord;
- }
-
- public void setPassWord(String passWord)
- {
- this.passWord = passWord;
- }
-
- public WriteConcern getWriteConcern()
- {
- return writeConcern;
- }
-
- public void setWriteConcern(WriteConcern writeConcern)
- {
- this.writeConcern = writeConcern;
- }
-
- private static final Logger logger = LoggerFactory.getLogger(MongoDBOutputOperator.class);
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/util/JsonUtils.java
----------------------------------------------------------------------
diff --git a/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/util/JsonUtils.java b/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/util/JsonUtils.java
deleted file mode 100644
index 60c200f..0000000
--- a/demos/frauddetect/src/main/java/com/datatorrent/demos/frauddetect/util/JsonUtils.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.frauddetect.util;
-
-import java.io.IOException;
-
-import org.codehaus.jackson.map.ObjectMapper;
-
-/**
- * Utility class to deal with JSON and Object
- *
- * @since 0.9.0
- */
-public class JsonUtils
-{
- private static final ObjectMapper mapper = new ObjectMapper();
-
- public static String toJson(Object obj) throws IOException
- {
- return mapper.writeValueAsString(obj);
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/frauddetect/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/demos/frauddetect/src/main/resources/META-INF/properties.xml b/demos/frauddetect/src/main/resources/META-INF/properties.xml
deleted file mode 100644
index a3a3073..0000000
--- a/demos/frauddetect/src/main/resources/META-INF/properties.xml
+++ /dev/null
@@ -1,167 +0,0 @@
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<configuration>
-<property>
- <name>dt.attr.STREAMING_WINDOW_SIZE_MILLIS</name>
- <value>1000</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.ccUserAlertQueryOutput.topic</name>
- <value>demos.app.frauddetect.fraudAlert</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.binUserAlertOutput.topic</name>
- <value>demos.app.frauddetect.fraudAlert</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.avgUserAlertQueryOutput.topic</name>
- <value>demos.app.frauddetect.fraudAlert</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.txSummaryWsOutput.topic</name>
- <value>demos.app.frauddetect.txSummary</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.bankInfoFraudDetector.attr.APPLICATION_WINDOW_COUNT
- </name>
- <value>10</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.bankInfoFraudDetector.threshold
- </name>
- <value>20</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.rangePerMerchant.attr.APPLICATION_WINDOW_COUNT
- </name>
- <value>1</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.txReceiver.attr.APPLICATION_WINDOW_COUNT
- </name>
- <value>1</value>
-</property>
-<!-- property>
- <name>dt.application.frauddetect.operator.smaPerMerchant.attr.APPLICATION_WINDOW_COUNT
- </name>
- <value>1</value>
-</property-->
-<property>
- <name>dt.application.FraudDetectDemo.operator.smaPerMerchant.windowSize
- </name>
- <value>30</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.movingSum.attr.APPLICATION_WINDOW_COUNT
- </name>
- <value>10</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.movingSum.windowSize
- </name>
- <value>3</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.avgAlerter.attr.APPLICATION_WINDOW_COUNT
- </name>
- <value>1</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.avgAlerter.threshold
- </name>
- <value>1200</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.amountFraudDetector.attr.APPLICATION_WINDOW_COUNT
- </name>
- <value>1</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.amountFraudDetector.threshold
- </name>
- <value>420</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.mongoTxStatsOutput.hostName</name>
- <value>localhost</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.mongoTxStatsOutput.dataBase</name>
- <value>frauddetect</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.mongoTxStatsOutput.collection</name>
- <value>txStats</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.mongoBinAlertsOutput.hostName</name>
- <value>localhost</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.mongoBinAlertsOutput.dataBase</name>
- <value>frauddetect</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.mongoBinAlertsOutput.collection</name>
- <value>binAlerts</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.mongoCcAlertsOutput.hostName</name>
- <value>localhost</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.mongoCcAlertsOutput.dataBase</name>
- <value>frauddetect</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.mongoCcAlertsOutput.collection</name>
- <value>ccAlerts</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.mongoAvgAlertsOutput.hostName</name>
- <value>localhost</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.mongoAvgAlertsOutput.dataBase</name>
- <value>frauddetect</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.mongoAvgAlertsOutput.collection</name>
- <value>avgAlerts</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.txStatsAggregator.attr.APPLICATION_WINDOW_COUNT
- </name>
- <value>1</value>
-</property>
-
-<property>
- <name>dt.application.FraudDetectDemo.port.*.attr.QUEUE_CAPACITY</name>
- <value>32000</value>
- </property>
- <property>
- <name>dt.application.FraudDetectDemo.operator.*.attr.MEMORY_MB</name>
- <value>2048</value>
- </property>
-
-</configuration>
-
-
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/frauddetect/src/site/conf/my-app-conf1.xml
----------------------------------------------------------------------
diff --git a/demos/frauddetect/src/site/conf/my-app-conf1.xml b/demos/frauddetect/src/site/conf/my-app-conf1.xml
deleted file mode 100644
index f35873b..0000000
--- a/demos/frauddetect/src/site/conf/my-app-conf1.xml
+++ /dev/null
@@ -1,27 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<configuration>
- <property>
- <name>dt.attr.MASTER_MEMORY_MB</name>
- <value>1024</value>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/frauddetect/src/test/java/com/datatorrent/demos/frauddetect/FrauddetectApplicationTest.java
----------------------------------------------------------------------
diff --git a/demos/frauddetect/src/test/java/com/datatorrent/demos/frauddetect/FrauddetectApplicationTest.java b/demos/frauddetect/src/test/java/com/datatorrent/demos/frauddetect/FrauddetectApplicationTest.java
deleted file mode 100644
index ef1f371..0000000
--- a/demos/frauddetect/src/test/java/com/datatorrent/demos/frauddetect/FrauddetectApplicationTest.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.frauddetect;
-
-import org.junit.Test;
-import org.apache.hadoop.conf.Configuration;
-import com.datatorrent.api.LocalMode;
-
-/**
- * Fraud detection application test
- */
-public class FrauddetectApplicationTest
-{
-
- public FrauddetectApplicationTest()
- {
- }
-
- @Test
- public void testApplication() throws Exception
- {
- try {
- Application application = new Application();
- Configuration conf = new Configuration(false);
- conf.addResource("dt-site-frauddetect.xml");
- LocalMode lma = LocalMode.newInstance();
- lma.prepareDAG(application, conf);
- lma.getController().run(120000);
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/frauddetect/src/test/resources/dt-site-frauddetect.xml
----------------------------------------------------------------------
diff --git a/demos/frauddetect/src/test/resources/dt-site-frauddetect.xml b/demos/frauddetect/src/test/resources/dt-site-frauddetect.xml
deleted file mode 100644
index 19771d8..0000000
--- a/demos/frauddetect/src/test/resources/dt-site-frauddetect.xml
+++ /dev/null
@@ -1,173 +0,0 @@
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<configuration>
- <property>
- <name>dt.application.FraudDetectDemo.class</name>
- <value>com.datatorrent.demos.frauddetect.Application</value>
- <description>An alias for the application</description>
-</property>
-<property>
- <name>dt.attr.STREAMING_WINDOW_SIZE_MILLIS</name>
- <value>1000</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.ccUserAlertQueryOutput.topic</name>
- <value>demos.app.frauddetect.fraudAlert</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.binUserAlertOutput.topic</name>
- <value>demos.app.frauddetect.fraudAlert</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.avgUserAlertQueryOutput.topic</name>
- <value>demos.app.frauddetect.fraudAlert</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.txSummaryWsOutput.topic</name>
- <value>demos.app.frauddetect.txSummary</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.bankInfoFraudDetector.attr.APPLICATION_WINDOW_COUNT
- </name>
- <value>10</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.bankInfoFraudDetector.threshold
- </name>
- <value>20</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.rangePerMerchant.attr.APPLICATION_WINDOW_COUNT
- </name>
- <value>1</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.txReceiver.attr.APPLICATION_WINDOW_COUNT
- </name>
- <value>1</value>
-</property>
-<!-- property>
- <name>dt.application.frauddetect.operator.smaPerMerchant.attr.APPLICATION_WINDOW_COUNT
- </name>
- <value>1</value>
-</property-->
-<property>
- <name>dt.application.FraudDetectDemo.operator.smaPerMerchant.windowSize
- </name>
- <value>30</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.movingSum.attr.APPLICATION_WINDOW_COUNT
- </name>
- <value>10</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.movingSum.windowSize
- </name>
- <value>3</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.avgAlerter.attr.APPLICATION_WINDOW_COUNT
- </name>
- <value>1</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.avgAlerter.threshold
- </name>
- <value>1200</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.amountFraudDetector.attr.APPLICATION_WINDOW_COUNT
- </name>
- <value>1</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.amountFraudDetector.threshold
- </name>
- <value>420</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.mongoTxStatsOutput.hostName</name>
- <value>localhost</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.mongoTxStatsOutput.dataBase</name>
- <value>frauddetect</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.mongoTxStatsOutput.collection</name>
- <value>txStats</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.mongoBinAlertsOutput.hostName</name>
- <value>localhost</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.mongoBinAlertsOutput.dataBase</name>
- <value>frauddetect</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.mongoBinAlertsOutput.collection</name>
- <value>binAlerts</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.mongoCcAlertsOutput.hostName</name>
- <value>localhost</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.mongoCcAlertsOutput.dataBase</name>
- <value>frauddetect</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.mongoCcAlertsOutput.collection</name>
- <value>ccAlerts</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.mongoAvgAlertsOutput.hostName</name>
- <value>localhost</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.mongoAvgAlertsOutput.dataBase</name>
- <value>frauddetect</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.mongoAvgAlertsOutput.collection</name>
- <value>avgAlerts</value>
-</property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.txStatsAggregator.attr.APPLICATION_WINDOW_COUNT
- </name>
- <value>1</value>
-</property>
-
-<property>
- <name>dt.application.FraudDetectDemo.port.*.attr.QUEUE_CAPACITY</name>
- <value>32000</value>
- </property>
-<property>
- <name>dt.application.FraudDetectDemo.operator.*.attr.MEMORY_MB</name>
- <value>2048</value>
-</property>
-
-
-</configuration>
-
-
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/frauddetect/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/demos/frauddetect/src/test/resources/log4j.properties b/demos/frauddetect/src/test/resources/log4j.properties
deleted file mode 100644
index cf0d19e..0000000
--- a/demos/frauddetect/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,43 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-log4j.rootLogger=DEBUG,CONSOLE
-
-log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
-log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
-log4j.appender.CONSOLE.threshold=${test.log.console.threshold}
-test.log.console.threshold=DEBUG
-
-log4j.appender.RFA=org.apache.log4j.RollingFileAppender
-log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
-log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
-log4j.appender.RFA.File=/tmp/app.log
-
-# to enable, add SYSLOG to rootLogger
-log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender
-log4j.appender.SYSLOG.syslogHost=127.0.0.1
-log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout
-log4j.appender.SYSLOG.layout.conversionPattern=${dt.cid} %-5p [%t] %c{2} %x - %m%n
-log4j.appender.SYSLOG.Facility=LOCAL1
-
-log4j.logger.org=info
-#log4j.logger.org.apache.commons.beanutils=warn
-log4j.logger.com.datatorrent=debug
-log4j.logger.org.apache.apex=debug
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/highlevelapi/pom.xml
----------------------------------------------------------------------
diff --git a/demos/highlevelapi/pom.xml b/demos/highlevelapi/pom.xml
deleted file mode 100644
index 1f25703..0000000
--- a/demos/highlevelapi/pom.xml
+++ /dev/null
@@ -1,141 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>high-level-api-demo</artifactId>
- <packaging>jar</packaging>
-
- <name>Apache Apex Malhar High-Level API Demo</name>
- <description>Apex demo applications that use High-level API to construct a dag</description>
-
- <parent>
- <groupId>org.apache.apex</groupId>
- <artifactId>malhar-demos</artifactId>
- <version>3.7.0-SNAPSHOT</version>
- </parent>
-
- <build>
- <plugins>
-
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <version>1.9.1</version>
- <executions>
- <execution>
- <id>attach-artifacts</id>
- <phase>package</phase>
- <goals>
- <goal>attach-artifact</goal>
- </goals>
- <configuration>
- <artifacts>
- <artifact>
- <file>target/${project.artifactId}-${project.version}.apa</file>
- <type>apa</type>
- </artifact>
- </artifacts>
- <skipAttach>false</skipAttach>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- </plugins>
- </build>
-
- <dependencies>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.10</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <!-- required by twitter demo -->
- <groupId>org.twitter4j</groupId>
- <artifactId>twitter4j-core</artifactId>
- <version>4.0.4</version>
- </dependency>
- <dependency>
- <!-- required by twitter demo -->
- <groupId>org.twitter4j</groupId>
- <artifactId>twitter4j-stream</artifactId>
- <version>4.0.4</version>
- </dependency>
- <dependency>
- <groupId>org.apache.apex</groupId>
- <artifactId>malhar-contrib</artifactId>
- <version>${project.version}</version>
- <exclusions>
- <exclusion>
- <groupId>*</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.apex</groupId>
- <artifactId>malhar-stream</artifactId>
- <version>${project.version}</version>
- <exclusions>
- <exclusion>
- <groupId>*</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.hsqldb</groupId>
- <artifactId>hsqldb</artifactId>
- <version>2.3.1</version>
- </dependency>
- <dependency>
- <groupId>com.h2database</groupId>
- <artifactId>h2</artifactId>
- <version>1.4.192</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>joda-time</groupId>
- <artifactId>joda-time</artifactId>
- <version>2.9.1</version>
- </dependency>
- <dependency>
- <!--This dependency is needed for StreamingWordExtractTest-->
- <groupId>org.codehaus.janino</groupId>
- <artifactId>commons-compiler</artifactId>
- <version>2.7.8</version>
- <type>jar</type>
- </dependency>
- <dependency>
- <!--This dependency is needed for StreamingWordExtractTest-->
- <groupId>org.codehaus.janino</groupId>
- <artifactId>janino</artifactId>
- <version>2.7.8</version>
- <scope>test</scope>
- </dependency>
- </dependencies>
-
-
-</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/highlevelapi/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/demos/highlevelapi/src/assemble/appPackage.xml b/demos/highlevelapi/src/assemble/appPackage.xml
deleted file mode 100644
index 4138cf2..0000000
--- a/demos/highlevelapi/src/assemble/appPackage.xml
+++ /dev/null
@@ -1,59 +0,0 @@
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
- <id>appPackage</id>
- <formats>
- <format>jar</format>
- </formats>
- <includeBaseDirectory>false</includeBaseDirectory>
- <fileSets>
- <fileSet>
- <directory>${basedir}/target/</directory>
- <outputDirectory>/app</outputDirectory>
- <includes>
- <include>${project.artifactId}-${project.version}.jar</include>
- </includes>
- </fileSet>
- <fileSet>
- <directory>${basedir}/target/deps</directory>
- <outputDirectory>/lib</outputDirectory>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/site/conf</directory>
- <outputDirectory>/conf</outputDirectory>
- <includes>
- <include>*.xml</include>
- </includes>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/main/resources/META-INF</directory>
- <outputDirectory>/META-INF</outputDirectory>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/main/resources/app</directory>
- <outputDirectory>/app</outputDirectory>
- </fileSet>
- </fileSets>
-
-</assembly>
-
[04/30] apex-malhar git commit: Renamed demos to examples. Packages
and artifactid names are changed as suggested.
Posted by th...@apache.org.
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/pom.xml
----------------------------------------------------------------------
diff --git a/examples/pom.xml b/examples/pom.xml
new file mode 100644
index 0000000..30ce061
--- /dev/null
+++ b/examples/pom.xml
@@ -0,0 +1,231 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar</artifactId>
+ <version>3.7.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>malhar-examples</artifactId>
+ <packaging>pom</packaging>
+ <name>Apache Apex Malhar Examples</name>
+
+ <properties>
+ <apex.apppackage.groupid>${project.groupId}</apex.apppackage.groupid>
+ <apex.apppackage.classpath>lib/*.jar</apex.apppackage.classpath>
+ <semver.plugin.skip>true</semver.plugin.skip>
+ <maven.deploy.skip>true</maven.deploy.skip>
+ </properties>
+
+ <profiles>
+ <profile>
+ <id>example-plugin-activation</id>
+ <activation>
+ <file>
+ <exists>${basedir}/src/main</exists>
+ </file>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-eclipse-plugin</artifactId>
+ <version>2.9</version>
+ <configuration>
+ <downloadSources>true</downloadSources>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>2.3.2</version>
+ <configuration>
+ <encoding>UTF-8</encoding>
+ <source>1.7</source>
+ <target>1.7</target>
+ <debug>true</debug>
+ <optimize>false</optimize>
+ <showDeprecation>true</showDeprecation>
+ <showWarnings>true</showWarnings>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <version>2.8</version>
+ <executions>
+ <execution>
+ <id>copy-dependencies</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>copy-dependencies</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>target/deps</outputDirectory>
+ <includeScope>runtime</includeScope>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>app-package-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <finalName>${project.artifactId}-${project.version}-apexapp</finalName>
+ <appendAssemblyId>false</appendAssemblyId>
+ <descriptors>
+ <descriptor>src/assemble/appPackage.xml</descriptor>
+ </descriptors>
+ <archiverConfig>
+ <defaultDirectoryMode>0755</defaultDirectoryMode>
+ </archiverConfig>
+ <archive>
+ <manifestEntries>
+ <Class-Path>${apex.apppackage.classpath}</Class-Path>
+ <DT-Engine-Version>${apex.core.version}</DT-Engine-Version>
+ <DT-App-Package-Group-Id>${apex.apppackage.groupid}</DT-App-Package-Group-Id>
+ <DT-App-Package-Name>${project.artifactId}</DT-App-Package-Name>
+ <DT-App-Package-Version>${project.version}</DT-App-Package-Version>
+ <DT-App-Package-Display-Name>${project.name}</DT-App-Package-Display-Name>
+ <DT-App-Package-Description>${project.description}</DT-App-Package-Description>
+ </manifestEntries>
+ </archive>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>1.7</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <configuration>
+ <target>
+ <move file="${project.build.directory}/${project.artifactId}-${project.version}-apexapp.jar"
+ tofile="${project.build.directory}/${project.artifactId}-${project.version}.apa" />
+ </target>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>1.9.1</version>
+ <executions>
+ <execution>
+ <id>attach-artifacts</id>
+ <phase>package</phase>
+ <goals>
+ <goal>attach-artifact</goal>
+ </goals>
+ <configuration>
+ <artifacts>
+ <artifact>
+ <file>target/${project.artifactId}-${project.version}.apa</file>
+ <type>apa</type>
+ </artifact>
+ </artifacts>
+ <skipAttach>false</skipAttach>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ <profile>
+ <id>all-modules</id>
+ <modules>
+ <module>distributedistinct</module>
+ <module>highlevelapi</module>
+ <module>sql</module>
+ </modules>
+ </profile>
+ </profiles>
+
+ <modules>
+ <module>machinedata</module>
+ <module>pi</module>
+ <module>twitter</module>
+ <module>yahoofinance</module>
+ <module>frauddetect</module>
+ <module>mobile</module>
+ <module>wordcount</module>
+ <module>mrmonitor</module>
+ <module>mroperator</module>
+ <module>uniquecount</module>
+ <module>r</module>
+ <module>echoserver</module>
+ <module>iteration</module>
+ </modules>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>apex-common</artifactId>
+ <version>${apex.core.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.10</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-library</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-library</artifactId>
+ <version>${project.version}</version>
+ <classifier>tests</classifier>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/r/pom.xml
----------------------------------------------------------------------
diff --git a/examples/r/pom.xml b/examples/r/pom.xml
new file mode 100644
index 0000000..46b7d47
--- /dev/null
+++ b/examples/r/pom.xml
@@ -0,0 +1,83 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>malhar-examples-r</artifactId>
+ <packaging>jar</packaging>
+
+ <name>Apache Apex Malhar R Example</name>
+ <description>Apex example applications for using R.</description>
+
+ <parent>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-examples</artifactId>
+ <version>3.7.0-SNAPSHOT</version>
+ </parent>
+
+ <properties>
+ <skipTests>true</skipTests>
+ </properties>
+
+ <repositories>
+ <repository>
+ <id>datatorrent-3rd-party</id>
+ <name>Embedded repository for dependencies not available online</name>
+ <url>https://www.datatorrent.com/maven/content/repositories/thirdparty</url>
+ <snapshots>
+ <updatePolicy>daily</updatePolicy>
+ </snapshots>
+ <releases>
+ <updatePolicy>daily</updatePolicy>
+ </releases>
+ </repository>
+ </repositories>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.rosuda</groupId>
+ <artifactId>jri</artifactId>
+ <version>1.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.rosuda</groupId>
+ <artifactId>rengine</artifactId>
+ <version>1.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.rosuda</groupId>
+ <artifactId>jriengine</artifactId>
+ <version>1.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-contrib</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/r/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/examples/r/src/assemble/appPackage.xml b/examples/r/src/assemble/appPackage.xml
new file mode 100644
index 0000000..4138cf2
--- /dev/null
+++ b/examples/r/src/assemble/appPackage.xml
@@ -0,0 +1,59 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+ <id>appPackage</id>
+ <formats>
+ <format>jar</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>${basedir}/target/</directory>
+ <outputDirectory>/app</outputDirectory>
+ <includes>
+ <include>${project.artifactId}-${project.version}.jar</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/target/deps</directory>
+ <outputDirectory>/lib</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/site/conf</directory>
+ <outputDirectory>/conf</outputDirectory>
+ <includes>
+ <include>*.xml</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/META-INF</directory>
+ <outputDirectory>/META-INF</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/app</directory>
+ <outputDirectory>/app</outputDirectory>
+ </fileSet>
+ </fileSets>
+
+</assembly>
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/r/src/main/java/org/apache/apex/examples/r/oldfaithful/FaithfulKey.java
----------------------------------------------------------------------
diff --git a/examples/r/src/main/java/org/apache/apex/examples/r/oldfaithful/FaithfulKey.java b/examples/r/src/main/java/org/apache/apex/examples/r/oldfaithful/FaithfulKey.java
new file mode 100755
index 0000000..8c08940
--- /dev/null
+++ b/examples/r/src/main/java/org/apache/apex/examples/r/oldfaithful/FaithfulKey.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.r.oldfaithful;
+
+/**
+ * @since 2.1.0
+ */
+public class FaithfulKey
+{
+
+ private static final long serialVersionUID = 201403251620L;
+
+ private double eruptionDuration;
+ private int waitingTime;
+
+ public FaithfulKey()
+ {
+ }
+
+ public double getEruptionDuration()
+ {
+ return eruptionDuration;
+ }
+
+ public void setEruptionDuration(double eruptionDuration)
+ {
+ this.eruptionDuration = eruptionDuration;
+ }
+
+ public int getWaitingTime()
+ {
+ return waitingTime;
+ }
+
+ public void setWaitingTime(int waitingTime)
+ {
+ this.waitingTime = waitingTime;
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/r/src/main/java/org/apache/apex/examples/r/oldfaithful/FaithfulRScript.java
----------------------------------------------------------------------
diff --git a/examples/r/src/main/java/org/apache/apex/examples/r/oldfaithful/FaithfulRScript.java b/examples/r/src/main/java/org/apache/apex/examples/r/oldfaithful/FaithfulRScript.java
new file mode 100755
index 0000000..4b61d42
--- /dev/null
+++ b/examples/r/src/main/java/org/apache/apex/examples/r/oldfaithful/FaithfulRScript.java
@@ -0,0 +1,111 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.r.oldfaithful;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.datatorrent.api.Context;
+import com.datatorrent.api.DefaultInputPort;
+import com.datatorrent.api.annotation.InputPortFieldAnnotation;
+import com.datatorrent.contrib.r.RScript;
+
+/**
+ * @since 2.1.0
+ */
+public class FaithfulRScript extends RScript
+{
+
+ private transient List<FaithfulKey> readingsList = new ArrayList<FaithfulKey>();
+ private int elapsedTime;
+ private static final Logger LOG = LoggerFactory.getLogger(FaithfulRScript.class);
+
+ public FaithfulRScript()
+ {
+ super();
+ }
+
+ public FaithfulRScript(String rScriptFilePath, String rFunction, String returnVariable)
+ {
+ super(rScriptFilePath, rFunction, returnVariable);
+ }
+
+ @InputPortFieldAnnotation(optional = true)
+ public final transient DefaultInputPort<FaithfulKey> faithfulInput = new DefaultInputPort<FaithfulKey>()
+ {
+ @Override
+ public void process(FaithfulKey tuple)
+ {
+ // Create a map of ("String", values) to be passed to the process
+ // function in the RScipt operator's process()
+ readingsList.add(tuple);
+
+ }
+
+ };
+
+ @InputPortFieldAnnotation(optional = true)
+ public final transient DefaultInputPort<Integer> inputElapsedTime = new DefaultInputPort<Integer>()
+ {
+ @Override
+ public void process(Integer eT)
+ {
+ elapsedTime = eT;
+ }
+ };
+
+ @Override
+ public void setup(Context.OperatorContext context)
+ {
+ super.setup(context);
+ }
+
+ @Override
+ public void endWindow()
+ {
+ if (readingsList.size() == 0) {
+ return;
+ }
+ LOG.info("Input data size: readingsList - " + readingsList.size());
+
+ double[] eruptionDuration = new double[readingsList.size()];
+ int[] waitingTime = new int[readingsList.size()];
+
+ for (int i = 0; i < readingsList.size(); i++) {
+ eruptionDuration[i] = readingsList.get(i).getEruptionDuration();
+ waitingTime[i] = readingsList.get(i).getWaitingTime();
+ }
+ LOG.info("Input data size: eruptionDuration - " + eruptionDuration.length);
+ LOG.info("Input data size: waitingTime - " + waitingTime.length);
+
+ HashMap<String, Object> map = new HashMap<String, Object>();
+
+ map.put("ELAPSEDTIME", elapsedTime);
+ map.put("ERUPTIONS", eruptionDuration);
+ map.put("WAITING", waitingTime);
+
+ super.process(map);
+ readingsList.clear();
+ map.clear();
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/r/src/main/java/org/apache/apex/examples/r/oldfaithful/InputGenerator.java
----------------------------------------------------------------------
diff --git a/examples/r/src/main/java/org/apache/apex/examples/r/oldfaithful/InputGenerator.java b/examples/r/src/main/java/org/apache/apex/examples/r/oldfaithful/InputGenerator.java
new file mode 100755
index 0000000..fb18726
--- /dev/null
+++ b/examples/r/src/main/java/org/apache/apex/examples/r/oldfaithful/InputGenerator.java
@@ -0,0 +1,145 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.r.oldfaithful;
+
+import java.util.Random;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.datatorrent.api.Context;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.InputOperator;
+
+/**
+ * The InputGenerator operator is used to generate input for the 'Old Faithful Geyser" application.
+ * This application accepts readings for the waiting time and the subsequent eruption duration
+ * of the 'Old Faithful' and based on this data, tries to predict the eruption duration of the next
+ * eruption given the elapsed time since the last eruption.
+ * The training data is generated for an application window and consists of multiple
+ * waiting times and eruption duration values.
+ * For every application window, it generates only one 'elapsed time' input for which the
+ * prediction would be made.
+ *
+ * @since 2.1.0
+ */
+
+public class InputGenerator implements InputOperator
+{
+
+ @SuppressWarnings("unused")
+ private static final Logger LOG = LoggerFactory.getLogger(InputGenerator.class);
+ private int blastCount = 1000;
+ private Random random = new Random();
+ private static int emitCount = 0;
+
+ public final transient DefaultOutputPort<FaithfulKey> outputPort = new DefaultOutputPort<FaithfulKey>();
+
+ public final transient DefaultOutputPort<Integer> elapsedTime = new DefaultOutputPort<Integer>();
+
+ public void setBlastCount(int blastCount)
+ {
+ this.blastCount = blastCount;
+ }
+
+ @Override
+ public void beginWindow(long windowId)
+ {
+ }
+
+ @Override
+ public void endWindow()
+ {
+ }
+
+ @Override
+ public void setup(Context.OperatorContext context)
+ {
+ }
+
+ @Override
+ public void teardown()
+ {
+ }
+
+ private int nextRandomId(int min, int max)
+ {
+ int id;
+ do {
+ id = (int)Math.abs(Math.round(random.nextGaussian() * max));
+ }
+ while (id >= max);
+
+ if (id < min) {
+ id = min;
+ }
+ try {
+ // Slowdown input generation
+ if (emitCount++ % 97 == 0) {
+ Thread.sleep(1);
+ }
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ return id;
+ }
+
+ @Override
+ public void emitTuples()
+ {
+ boolean elapsedTimeSent = false;
+
+ try {
+ for (int i = 0; i < blastCount; ++i) {
+ int waitingTime = nextRandomId(3600, 36000);
+
+ double eruptionDuration = -2.15 + 0.05 * waitingTime;
+ emitTuple(eruptionDuration, waitingTime);
+
+ if (!elapsedTimeSent) {
+ int eT = 0;
+
+ if (i % 100 == 0) {
+ eT = 54 + waitingTime;
+
+ emitElapsedTime(eT);
+ elapsedTimeSent = true;
+ }
+ }
+ }
+ } catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+
+ private void emitTuple(double eruptionDuration, int waitingTime)
+ {
+ FaithfulKey faithfulkey = new FaithfulKey();
+
+ faithfulkey.setEruptionDuration(eruptionDuration);
+ faithfulkey.setWaitingTime(waitingTime);
+
+ this.outputPort.emit(faithfulkey);
+ }
+
+ private void emitElapsedTime(int eT)
+ {
+ this.elapsedTime.emit(eT);
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/r/src/main/java/org/apache/apex/examples/r/oldfaithful/OldFaithfulApplication.java
----------------------------------------------------------------------
diff --git a/examples/r/src/main/java/org/apache/apex/examples/r/oldfaithful/OldFaithfulApplication.java b/examples/r/src/main/java/org/apache/apex/examples/r/oldfaithful/OldFaithfulApplication.java
new file mode 100755
index 0000000..bd51c29
--- /dev/null
+++ b/examples/r/src/main/java/org/apache/apex/examples/r/oldfaithful/OldFaithfulApplication.java
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.r.oldfaithful;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.lib.io.ConsoleOutputOperator;
+
+/**
+ * The application attempts to simulate 'Old Faithful Geyser" eruption.
+ * This application accepts readings for the waiting time and the subsequent eruption duration
+ * of the 'Old Faithful' and based on this data, tries to predict the eruption duration of the next
+ * eruption given the elapsed time since the last eruption.
+ * The training data is generated for an application window and consists of multiple
+ * waiting times and eruption duration values.
+ * For every application window, it generates only one 'elapsed time' input for which the
+ * prediction would be made.
+ * Model in R is in file ruptionModel.R located at
+ * examples/r/src/main/resources/com/datatorrent/examples/oldfaithful/ directory
+ *
+ * @since 2.1.0
+ */
+
+@ApplicationAnnotation(name = "OldFaithfulApplication")
+public class OldFaithfulApplication implements StreamingApplication
+{
+ private final DAG.Locality locality = null;
+
+ /**
+ * Create the DAG
+ */
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+
+ InputGenerator randomInputGenerator = dag.addOperator("rand", new InputGenerator());
+ FaithfulRScript rScriptOp = dag.addOperator("rScriptOp", new FaithfulRScript("com/datatorrent/examples/r/oldfaithful/eruptionModel.R", "eruptionModel", "retVal"));
+ ConsoleOutputOperator consoles = dag.addOperator("consoles", new ConsoleOutputOperator());
+
+ Map<String, FaithfulRScript.REXP_TYPE> argTypeMap = new HashMap<String, FaithfulRScript.REXP_TYPE>();
+
+ argTypeMap.put("ELAPSEDTIME", FaithfulRScript.REXP_TYPE.REXP_INT);
+ argTypeMap.put("ERUPTIONS", FaithfulRScript.REXP_TYPE.REXP_ARRAY_DOUBLE);
+ argTypeMap.put("WAITING", FaithfulRScript.REXP_TYPE.REXP_ARRAY_INT);
+
+ rScriptOp.setArgTypeMap(argTypeMap);
+
+ dag.addStream("ingen_faithfulRscript", randomInputGenerator.outputPort, rScriptOp.faithfulInput).setLocality(locality);
+ dag.addStream("ingen_faithfulRscript_eT", randomInputGenerator.elapsedTime, rScriptOp.inputElapsedTime).setLocality(locality);
+ dag.addStream("faithfulRscript_console_s", rScriptOp.strOutput, consoles.input).setLocality(locality);
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/r/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/examples/r/src/main/resources/META-INF/properties.xml b/examples/r/src/main/resources/META-INF/properties.xml
new file mode 100755
index 0000000..07c1e87
--- /dev/null
+++ b/examples/r/src/main/resources/META-INF/properties.xml
@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<configuration>
+<!--properties for R example -->
+ <property>
+ <name>dt.application.OldFaithfulApplication.class</name>
+ <value>org.apache.apex.examples.r.oldfaithful.OldFaithfulApplication</value>
+ <description>An alias for OldFaithful application</description>
+ </property>
+
+ <property>
+ <name>dt.application.OldFaithfulApplication.operator.*.attr.MEMORY_MB</name>
+ <value>1024</value>
+ </property>
+
+<!-- Need this to information for loading native libraries -->
+ <property>
+ <name>dt.attr.CONTAINER_JVM_OPTIONS</name>
+ <value>-Djava.library.path=/usr/local/lib/R/site-library/rJava/jri/</value>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/r/src/main/resources/org/apache/apex/examples/r/oldfaithful/eruptionModel.R
----------------------------------------------------------------------
diff --git a/examples/r/src/main/resources/org/apache/apex/examples/r/oldfaithful/eruptionModel.R b/examples/r/src/main/resources/org/apache/apex/examples/r/oldfaithful/eruptionModel.R
new file mode 100755
index 0000000..e46fa8d
--- /dev/null
+++ b/examples/r/src/main/resources/org/apache/apex/examples/r/oldfaithful/eruptionModel.R
@@ -0,0 +1,60 @@
+#!/usr/bin/Rscript
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+
+# This script apply the simple linear regression model for the data set 'faithful',
+# and estimates the next eruption duration given the waiting time since the last eruption.
+#
+
+ eruptionModel <- function() {
+
+ datavar = data.frame(ERUPTIONS, WAITING)
+
+ #attach data variable
+ attach(datavar)
+
+ #create a linear model using lm(FORMULA, DATAVAR)
+ #predict the fall eruption duration (ERUPT) using the waiting time since the last eruption (WAITING)
+ eruption.lm <- lm(ERUPTIONS ~ WAITING, datavar)
+
+ #display linear model
+ eruption.lm
+
+ # Get the values of the intercept and unemployment so as to be able to predict the enrolment
+ interc<-eruption.lm$coeff[["(Intercept)"]]
+ eruptionDuration<-eruption.lm$coeff[["WAITING"]]
+
+ # Calculate the enrollment based on the percentage being asked for, and the model that has been rated above.
+ nextEruptionDuration<-(interc+(eruptionDuration * ELAPSEDTIME))
+
+retVal<-paste("nextEruptionDuration ", nextEruptionDuration, sep=": ")
+#retVal<-c("interc : ",interc, ", eruptionDuration : ", eruptionDuration,", nextEruptionDuration : ", nextEruptionDuration)
+
+sort( sapply(mget(ls()),object.size) )
+
+detach(datavar);
+
+# Clear all the data from R workspace
+rm(datavar);
+rm(ERUPTIONS);
+rm(WAITING);
+
+return(retVal)
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/r/src/test/java/org/apache/apex/examples/r/oldfaithful/OldFaithfulApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/r/src/test/java/org/apache/apex/examples/r/oldfaithful/OldFaithfulApplicationTest.java b/examples/r/src/test/java/org/apache/apex/examples/r/oldfaithful/OldFaithfulApplicationTest.java
new file mode 100755
index 0000000..0ebe958
--- /dev/null
+++ b/examples/r/src/test/java/org/apache/apex/examples/r/oldfaithful/OldFaithfulApplicationTest.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.r.oldfaithful;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.LocalMode;
+
+public class OldFaithfulApplicationTest
+{
+
+ private static final Logger LOG = LoggerFactory.getLogger(OldFaithfulApplicationTest.class);
+
+ @Test
+ public void testSomeMethod() throws Exception
+ {
+ LocalMode lma = LocalMode.newInstance();
+ OldFaithfulApplication app = new OldFaithfulApplication();
+ app.populateDAG(lma.getDAG(), new Configuration(false));
+
+ try {
+ LocalMode.Controller lc = lma.getController();
+ lc.setHeartbeatMonitoringEnabled(false);
+ lc.run(5000);
+ } catch (Exception e) {
+ LOG.error("Exception: ", e);
+ Assert.fail("Unexpected exception.");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/r/src/test/resources/dt-site-oldfaithful.xml
----------------------------------------------------------------------
diff --git a/examples/r/src/test/resources/dt-site-oldfaithful.xml b/examples/r/src/test/resources/dt-site-oldfaithful.xml
new file mode 100755
index 0000000..07c1e87
--- /dev/null
+++ b/examples/r/src/test/resources/dt-site-oldfaithful.xml
@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<configuration>
+<!--properties for R example -->
+ <property>
+ <name>dt.application.OldFaithfulApplication.class</name>
+ <value>org.apache.apex.examples.r.oldfaithful.OldFaithfulApplication</value>
+ <description>An alias for OldFaithful application</description>
+ </property>
+
+ <property>
+ <name>dt.application.OldFaithfulApplication.operator.*.attr.MEMORY_MB</name>
+ <value>1024</value>
+ </property>
+
+<!-- Need this to information for loading native libraries -->
+ <property>
+ <name>dt.attr.CONTAINER_JVM_OPTIONS</name>
+ <value>-Djava.library.path=/usr/local/lib/R/site-library/rJava/jri/</value>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/r/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/examples/r/src/test/resources/log4j.properties b/examples/r/src/test/resources/log4j.properties
new file mode 100755
index 0000000..cf0d19e
--- /dev/null
+++ b/examples/r/src/test/resources/log4j.properties
@@ -0,0 +1,43 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+log4j.rootLogger=DEBUG,CONSOLE
+
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+log4j.appender.CONSOLE.threshold=${test.log.console.threshold}
+test.log.console.threshold=DEBUG
+
+log4j.appender.RFA=org.apache.log4j.RollingFileAppender
+log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
+log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+log4j.appender.RFA.File=/tmp/app.log
+
+# to enable, add SYSLOG to rootLogger
+log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender
+log4j.appender.SYSLOG.syslogHost=127.0.0.1
+log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout
+log4j.appender.SYSLOG.layout.conversionPattern=${dt.cid} %-5p [%t] %c{2} %x - %m%n
+log4j.appender.SYSLOG.Facility=LOCAL1
+
+log4j.logger.org=info
+#log4j.logger.org.apache.commons.beanutils=warn
+log4j.logger.com.datatorrent=debug
+log4j.logger.org.apache.apex=debug
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/sql/pom.xml
----------------------------------------------------------------------
diff --git a/examples/sql/pom.xml b/examples/sql/pom.xml
new file mode 100644
index 0000000..7eb0f4d
--- /dev/null
+++ b/examples/sql/pom.xml
@@ -0,0 +1,102 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>malhar-examples-sql</artifactId>
+ <packaging>jar</packaging>
+
+ <name>Apache Apex Malhar SQL API Example</name>
+ <description>Apex example applications that use SQL APIs to construct a DAG</description>
+
+ <parent>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-examples</artifactId>
+ <version>3.7.0-SNAPSHOT</version>
+ </parent>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>1.9.1</version>
+ <executions>
+ <execution>
+ <id>attach-artifacts</id>
+ <phase>package</phase>
+ <goals>
+ <goal>attach-artifact</goal>
+ </goals>
+ <configuration>
+ <artifacts>
+ <artifact>
+ <file>target/${project.artifactId}-${project.version}.apa</file>
+ <type>apa</type>
+ </artifact>
+ </artifacts>
+ <skipAttach>false</skipAttach>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ </plugins>
+ </build>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>apex-engine</artifactId>
+ <version>${apex.core.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-sql</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <!-- For KafkaTest -->
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-kafka</artifactId>
+ <version>${project.parent.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka_2.11</artifactId>
+ <version>0.9.0.0</version>
+ <classifier>test</classifier>
+ <scope>test</scope>
+ </dependency>
+
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/sql/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/examples/sql/src/assemble/appPackage.xml b/examples/sql/src/assemble/appPackage.xml
new file mode 100644
index 0000000..4138cf2
--- /dev/null
+++ b/examples/sql/src/assemble/appPackage.xml
@@ -0,0 +1,59 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+ <id>appPackage</id>
+ <formats>
+ <format>jar</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>${basedir}/target/</directory>
+ <outputDirectory>/app</outputDirectory>
+ <includes>
+ <include>${project.artifactId}-${project.version}.jar</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/target/deps</directory>
+ <outputDirectory>/lib</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/site/conf</directory>
+ <outputDirectory>/conf</outputDirectory>
+ <includes>
+ <include>*.xml</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/META-INF</directory>
+ <outputDirectory>/META-INF</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/app</directory>
+ <outputDirectory>/app</outputDirectory>
+ </fileSet>
+ </fileSets>
+
+</assembly>
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/sql/src/main/java/org/apache/apex/malhar/sql/sample/FusionStyleSQLApplication.java
----------------------------------------------------------------------
diff --git a/examples/sql/src/main/java/org/apache/apex/malhar/sql/sample/FusionStyleSQLApplication.java b/examples/sql/src/main/java/org/apache/apex/malhar/sql/sample/FusionStyleSQLApplication.java
new file mode 100644
index 0000000..80b997d
--- /dev/null
+++ b/examples/sql/src/main/java/org/apache/apex/malhar/sql/sample/FusionStyleSQLApplication.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.malhar.sql.sample;
+
+import java.util.Date;
+import java.util.Map;
+
+import org.apache.apex.malhar.kafka.KafkaSinglePortInputOperator;
+import org.apache.apex.malhar.sql.SQLExecEnvironment;
+import org.apache.apex.malhar.sql.table.CSVMessageFormat;
+import org.apache.apex.malhar.sql.table.FileEndpoint;
+import org.apache.apex.malhar.sql.table.StreamEndpoint;
+import org.apache.hadoop.conf.Configuration;
+
+import com.google.common.collect.ImmutableMap;
+
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.DefaultInputPort;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.common.util.BaseOperator;
+import com.datatorrent.contrib.parser.CsvParser;
+
+
+@ApplicationAnnotation(name = "FusionStyleSQLApplication")
+/**
+ * @since 3.6.0
+ */
+public class FusionStyleSQLApplication implements StreamingApplication
+{
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ SQLExecEnvironment env = SQLExecEnvironment.getEnvironment();
+ env.registerFunction("APEXCONCAT", PureStyleSQLApplication.class, "apex_concat_str");
+
+ Map<String, Class> fieldMapping = ImmutableMap.<String, Class>of(
+ "RowTime", Date.class,
+ "id", Integer.class,
+ "Product", String.class,
+ "units", Integer.class);
+
+ // Add Kafka Input
+ KafkaSinglePortInputOperator kafkaInput = dag.addOperator("KafkaInput", KafkaSinglePortInputOperator.class);
+ kafkaInput.setInitialOffset("EARLIEST");
+
+ // Add CSVParser
+ CsvParser csvParser = dag.addOperator("CSVParser", CsvParser.class);
+ dag.addStream("KafkaToCSV", kafkaInput.outputPort, csvParser.in);
+
+ // Register CSV Parser output as input table for first SQL
+ env.registerTable(conf.get("sqlSchemaInputName"), new StreamEndpoint(csvParser.out, fieldMapping));
+
+ // Register FileEndpoint as output table for second SQL.
+ env.registerTable(conf.get("sqlSchemaOutputName"), new FileEndpoint(conf.get("folderPath"),
+ conf.get("fileName"), new CSVMessageFormat(conf.get("sqlSchemaOutputDef"))));
+
+ // Add second SQL to DAG
+ env.executeSQL(dag, conf.get("sql"));
+ }
+
+ public static class PassThroughOperator extends BaseOperator
+ {
+ public final transient DefaultOutputPort output = new DefaultOutputPort();
+ public final transient DefaultInputPort input = new DefaultInputPort()
+ {
+ @Override
+ public void process(Object o)
+ {
+ output.emit(output);
+ }
+ };
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/sql/src/main/java/org/apache/apex/malhar/sql/sample/PureStyleSQLApplication.java
----------------------------------------------------------------------
diff --git a/examples/sql/src/main/java/org/apache/apex/malhar/sql/sample/PureStyleSQLApplication.java b/examples/sql/src/main/java/org/apache/apex/malhar/sql/sample/PureStyleSQLApplication.java
new file mode 100644
index 0000000..79295f9
--- /dev/null
+++ b/examples/sql/src/main/java/org/apache/apex/malhar/sql/sample/PureStyleSQLApplication.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.malhar.sql.sample;
+
+import org.apache.apex.malhar.sql.SQLExecEnvironment;
+import org.apache.apex.malhar.sql.table.CSVMessageFormat;
+import org.apache.apex.malhar.sql.table.FileEndpoint;
+import org.apache.apex.malhar.sql.table.KafkaEndpoint;
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+
+@ApplicationAnnotation(name = "PureStyleSQLApplication")
+/**
+ * @since 3.6.0
+ */
+public class PureStyleSQLApplication implements StreamingApplication
+{
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ // Source definition
+ String schemaInName = conf.get("schemaInName");
+ String schemaInDef = conf.get("schemaInDef");
+ String broker = conf.get("broker");
+ String sourceTopic = conf.get("topic");
+
+ // Destination definition
+ String schemaOutName = conf.get("schemaOutName");
+ String schemaOutDef = conf.get("schemaOutDef");
+ String outputFolder = conf.get("outputFolder");
+ String outFilename = conf.get("destFileName");
+
+ // SQL statement
+ String sql = conf.get("sql");
+
+ SQLExecEnvironment.getEnvironment()
+ .registerTable(schemaInName, new KafkaEndpoint(broker, sourceTopic,
+ new CSVMessageFormat(schemaInDef)))
+ .registerTable(schemaOutName, new FileEndpoint(outputFolder, outFilename,
+ new CSVMessageFormat(schemaOutDef)))
+ .registerFunction("APEXCONCAT", this.getClass(), "apex_concat_str")
+ .executeSQL(dag, sql);
+ }
+
+ public static String apex_concat_str(String s1, String s2)
+ {
+ return s1 + s2;
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/sql/src/main/java/org/apache/apex/malhar/sql/sample/SQLApplicationWithAPI.java
----------------------------------------------------------------------
diff --git a/examples/sql/src/main/java/org/apache/apex/malhar/sql/sample/SQLApplicationWithAPI.java b/examples/sql/src/main/java/org/apache/apex/malhar/sql/sample/SQLApplicationWithAPI.java
new file mode 100644
index 0000000..da4f563
--- /dev/null
+++ b/examples/sql/src/main/java/org/apache/apex/malhar/sql/sample/SQLApplicationWithAPI.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.malhar.sql.sample;
+
+import org.apache.apex.malhar.sql.SQLExecEnvironment;
+import org.apache.apex.malhar.sql.table.CSVMessageFormat;
+import org.apache.apex.malhar.sql.table.FileEndpoint;
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+
+@ApplicationAnnotation(name = "SQLApplicationWithAPI")
+/**
+ * @since 3.6.0
+ */
+public class SQLApplicationWithAPI implements StreamingApplication
+{
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ // Source definition
+ String schemaInName = conf.get("csvSchemaInName");
+ String schemaIn = conf.get("csvSchemaIn");
+ String sourceFile = conf.get("sourceFile");
+
+ SQLExecEnvironment.getEnvironment()
+ .registerTable(schemaInName, new FileEndpoint(sourceFile, new CSVMessageFormat(schemaIn)))
+ .executeSQL(dag, conf.get("sql"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/sql/src/main/java/org/apache/apex/malhar/sql/sample/SQLApplicationWithModelFile.java
----------------------------------------------------------------------
diff --git a/examples/sql/src/main/java/org/apache/apex/malhar/sql/sample/SQLApplicationWithModelFile.java b/examples/sql/src/main/java/org/apache/apex/malhar/sql/sample/SQLApplicationWithModelFile.java
new file mode 100644
index 0000000..4c90a82
--- /dev/null
+++ b/examples/sql/src/main/java/org/apache/apex/malhar/sql/sample/SQLApplicationWithModelFile.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.malhar.sql.sample;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.apex.malhar.sql.SQLExecEnvironment;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+
+@ApplicationAnnotation(name = "SQLApplicationWithModelFile")
+/**
+ * @since 3.6.0
+ */
+public class SQLApplicationWithModelFile implements StreamingApplication
+{
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ String modelFile = conf.get("modelFile");
+ String model;
+ try {
+ model = FileUtils.readFileToString(new File(modelFile));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ SQLExecEnvironment.getEnvironment()
+ .withModel(model)
+ .executeSQL(dag, conf.get("sql"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/sql/src/main/resources/META-INF/properties-FusionStyleSQLApplication.xml
----------------------------------------------------------------------
diff --git a/examples/sql/src/main/resources/META-INF/properties-FusionStyleSQLApplication.xml b/examples/sql/src/main/resources/META-INF/properties-FusionStyleSQLApplication.xml
new file mode 100644
index 0000000..77852e7
--- /dev/null
+++ b/examples/sql/src/main/resources/META-INF/properties-FusionStyleSQLApplication.xml
@@ -0,0 +1,65 @@
+<?xml version="1.0"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<configuration>
+ <!-- Kafka Operator Properties -->
+ <property>
+ <name>dt.operator.KafkaInput.prop.topics</name>
+ <value>dataTopic</value>
+ </property>
+ <property>
+ <name>dt.operator.KafkaInput.prop.clusters</name>
+ <value>localhost:9092</value> <!-- broker (NOT zookeeper) address -->
+ </property>
+
+ <!-- CSV Parser Properties -->
+ <property>
+ <name>dt.operator.CSVParser.prop.schema</name>
+ <value>{"separator":",","quoteChar":"\"","fields":[{"name":"RowTime","type":"Date","constraints":{"format":"dd/MM/yyyy hh:mm:ss Z"}},{"name":"id","type":"Integer"},{"name":"Product","type":"String"},{"name":"units","type":"Integer"}]}</value>
+ </property>
+
+ <!-- SQL Properties -->
+ <property>
+ <name>sqlSchemaInputName</name>
+ <value>FROMCSV</value>
+ </property>
+ <property>
+ <name>sqlSchemaOutputName</name>
+ <value>TOFILE</value>
+ </property>
+ <property>
+ <name>folderPath</name>
+ <value>/tmp/output</value>
+ </property>
+ <property>
+ <name>fileName</name>
+ <value>output.txt</value>
+ </property>
+ <property>
+ <name>sqlSchemaOutputDef</name>
+ <value>{"separator":",","quoteChar":"\"","fields":[{"name":"RowTime1","type":"Date","constraints":{"format":"dd/MM/yyyy hh:mm:ss Z"}},{"name":"RowTime2","type":"Date","constraints":{"format":"dd/MM/yyyy hh:mm:ss Z"}},{"name":"Product","type":"String"}]}</value>
+ </property>
+ <property>
+ <name>sql</name>
+ <value>INSERT INTO TOFILE SELECT STREAM ROWTIME, FLOOR(ROWTIME TO DAY), APEXCONCAT('OILPAINT', SUBSTRING(PRODUCT, 6, 7)) FROM FROMCSV WHERE ID > 3 AND PRODUCT LIKE 'paint%'</value>
+ </property>
+</configuration>
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/sql/src/main/resources/META-INF/properties-PureStyleSQLApplication.xml
----------------------------------------------------------------------
diff --git a/examples/sql/src/main/resources/META-INF/properties-PureStyleSQLApplication.xml b/examples/sql/src/main/resources/META-INF/properties-PureStyleSQLApplication.xml
new file mode 100644
index 0000000..0d25aa6
--- /dev/null
+++ b/examples/sql/src/main/resources/META-INF/properties-PureStyleSQLApplication.xml
@@ -0,0 +1,65 @@
+<?xml version="1.0"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<configuration>
+ <!-- Input Definition -->
+ <property>
+ <name>schemaInName</name>
+ <value>ORDERS</value>
+ </property>
+ <property>
+ <name>schemaInDef</name>
+ <value>{"separator":",","quoteChar":"\"","fields":[{"name":"RowTime","type":"Date","constraints":{"format":"dd/MM/yyyy hh:mm:ss Z"}},{"name":"id","type":"Integer"},{"name":"Product","type":"String"},{"name":"units","type":"Integer"}]}</value>
+ </property>
+ <property>
+ <name>broker</name>
+ <value>localhost:9090</value>
+ </property>
+ <property>
+ <name>topic</name>
+ <value>inputTopic</value>
+ </property>
+
+ <!-- Output Definition -->
+ <property>
+ <name>schemaOutName</name>
+ <value>SALES</value>
+ </property>
+ <property>
+ <name>schemaOutDef</name>
+ <value>{"separator":",","quoteChar":"\"","fields":[{"name":"RowTime1","type":"Date","constraints":{"format":"dd/MM/yyyy hh:mm:ss Z"}},{"name":"RowTime2","type":"Date","constraints":{"format":"dd/MM/yyyy hh:mm:ss Z"}},{"name":"Product","type":"String"}]}</value>
+ </property>
+ <property>
+ <name>outputFolder</name>
+ <value>/tmp/output</value>
+ </property>
+ <property>
+ <name>destFileName</name>
+ <value>out.file</value>
+ </property>
+
+ <!-- Execution SQL -->
+ <property>
+ <name>sql</name>
+ <value>INSERT INTO SALES SELECT STREAM ROWTIME, FLOOR(ROWTIME TO DAY), APEXCONCAT('OILPAINT', SUBSTRING(PRODUCT, 6, 7)) FROM ORDERS WHERE ID > 3 AND PRODUCT LIKE 'paint%'</value>
+ </property>
+</configuration>
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/sql/src/main/resources/META-INF/properties-SQLApplicationWithAPI.xml
----------------------------------------------------------------------
diff --git a/examples/sql/src/main/resources/META-INF/properties-SQLApplicationWithAPI.xml b/examples/sql/src/main/resources/META-INF/properties-SQLApplicationWithAPI.xml
new file mode 100644
index 0000000..9ac49d4
--- /dev/null
+++ b/examples/sql/src/main/resources/META-INF/properties-SQLApplicationWithAPI.xml
@@ -0,0 +1,43 @@
+<?xml version="1.0"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<configuration>
+ <!-- Input Definition -->
+ <property>
+ <name>csvSchemaInName</name>
+ <value>ORDERS</value>
+ </property>
+ <property>
+ <name>csvSchemaIn</name>
+ <value>{"separator":",","quoteChar":"\"","fields":[{"name":"RowTime","type":"Date","constraints":{"format":"dd/MM/yyyy hh:mm:ss Z"}},{"name":"id","type":"Integer"},{"name":"Product","type":"String"},{"name":"units","type":"Integer"}]}</value>
+ </property>
+ <property>
+ <name>sourceFile</name>
+ <value>src/test/resources/input.csv</value>
+ </property>
+
+ <!-- Execution SQL -->
+ <property>
+ <name>sql</name>
+ <value>SELECT STREAM ROWTIME, PRODUCT FROM ORDERS</value>
+ </property>
+</configuration>
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/sql/src/main/resources/META-INF/properties-SQLApplicationWithModelFile.xml
----------------------------------------------------------------------
diff --git a/examples/sql/src/main/resources/META-INF/properties-SQLApplicationWithModelFile.xml b/examples/sql/src/main/resources/META-INF/properties-SQLApplicationWithModelFile.xml
new file mode 100644
index 0000000..ab026c2
--- /dev/null
+++ b/examples/sql/src/main/resources/META-INF/properties-SQLApplicationWithModelFile.xml
@@ -0,0 +1,32 @@
+<?xml version="1.0"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<configuration>
+ <property>
+ <name>modelFile</name>
+ <value>src/main/resources/model/model_file_csv.json</value>
+ </property>
+ <property>
+ <name>sql</name>
+ <value>SELECT STREAM ROWTIME, PRODUCT FROM ORDERS</value>
+ </property>
+</configuration>
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/sql/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/examples/sql/src/main/resources/META-INF/properties.xml b/examples/sql/src/main/resources/META-INF/properties.xml
new file mode 100644
index 0000000..2702315
--- /dev/null
+++ b/examples/sql/src/main/resources/META-INF/properties.xml
@@ -0,0 +1,41 @@
+<?xml version="1.0"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<configuration>
+ <!-- Memory settings for all examples -->
+ <property>
+ <name>dt.attr.MASTER_MEMORY_MB</name>
+ <value>512</value>
+ </property>
+ <property>
+ <name>dt.application.*.operator.*.attr.MEMORY_MB</name>
+ <value>256</value>
+ </property>
+ <property>
+ <name>dt.application.*.operator.*.attr.JVM_OPTIONS</name>
+ <value>-Xmx128M</value>
+ </property>
+ <property>
+ <name>dt.application.*.operator.*.port.*.attr.BUFFER_MEMORY_MB</name>
+ <value>128</value>
+ </property>
+</configuration>
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/sql/src/main/resources/model/model_file_csv.json
----------------------------------------------------------------------
diff --git a/examples/sql/src/main/resources/model/model_file_csv.json b/examples/sql/src/main/resources/model/model_file_csv.json
new file mode 100644
index 0000000..beba18d
--- /dev/null
+++ b/examples/sql/src/main/resources/model/model_file_csv.json
@@ -0,0 +1,27 @@
+{
+ "version": "1.0",
+ "defaultSchema": "APEX",
+ "schemas": [{
+ "name": "APEX",
+ "tables": [
+ {
+ "name": "ORDERS",
+ "type": "custom",
+ "factory": "org.apache.apex.malhar.sql.schema.ApexSQLTableFactory",
+ "stream": {
+ "stream": true
+ },
+ "operand": {
+ "endpoint": "file",
+ "messageFormat": "csv",
+ "endpointOperands": {
+ "directory": "src/test/resources/input.csv"
+ },
+ "messageFormatOperands": {
+ "schema": "{\"separator\":\",\",\"quoteChar\":\"\\\"\",\"fields\":[{\"name\":\"RowTime\",\"type\":\"Date\",\"constraints\":{\"format\":\"dd/MM/yyyy hh:mm:ss\"}},{\"name\":\"id\",\"type\":\"Integer\"},{\"name\":\"Product\",\"type\":\"String\"},{\"name\":\"units\",\"type\":\"Integer\"}]}"
+ }
+ }
+ }
+ ]
+ }]
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/sql/src/test/java/org/apache/apex/malhar/sql/sample/FusionStyleSQLApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/sql/src/test/java/org/apache/apex/malhar/sql/sample/FusionStyleSQLApplicationTest.java b/examples/sql/src/test/java/org/apache/apex/malhar/sql/sample/FusionStyleSQLApplicationTest.java
new file mode 100644
index 0000000..7208701
--- /dev/null
+++ b/examples/sql/src/test/java/org/apache/apex/malhar/sql/sample/FusionStyleSQLApplicationTest.java
@@ -0,0 +1,121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.malhar.sql.sample;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.List;
+import java.util.TimeZone;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+
+import org.apache.apex.malhar.kafka.EmbeddedKafka;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.LocalMode;
+
+public class FusionStyleSQLApplicationTest
+{
+ private final String testTopicData = "dataTopic";
+ private final String testTopicResult = "resultTopic";
+
+ private TimeZone defaultTZ;
+ private EmbeddedKafka kafka;
+
+ private static String outputFolder = "target/output/";
+
+ @Rule
+ public TestName testName = new TestName();
+
+ @Before
+ public void setUp() throws Exception
+ {
+ defaultTZ = TimeZone.getDefault();
+ TimeZone.setDefault(TimeZone.getTimeZone("GMT"));
+
+ kafka = new EmbeddedKafka();
+ kafka.start();
+ kafka.createTopic(testTopicData);
+ kafka.createTopic(testTopicResult);
+
+ outputFolder += testName.getMethodName() + "/";
+ }
+
+ @After
+ public void tearDown() throws Exception
+ {
+ kafka.stop();
+
+ TimeZone.setDefault(defaultTZ);
+ }
+
+ @Test
+ public void test() throws Exception
+ {
+ try {
+ LocalMode lma = LocalMode.newInstance();
+ Configuration conf = new Configuration(false);
+ conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
+ conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties-FusionStyleSQLApplication.xml"));
+
+ conf.set("dt.operator.KafkaInput.prop.topics", testTopicData);
+ conf.set("dt.operator.KafkaInput.prop.clusters", kafka.getBroker());
+ conf.set("folderPath", outputFolder);
+ conf.set("fileName", "out.tmp");
+
+ FusionStyleSQLApplication app = new FusionStyleSQLApplication();
+
+ lma.prepareDAG(app, conf);
+
+ LocalMode.Controller lc = lma.getController();
+
+ lc.runAsync();
+ kafka.publish(testTopicData, Arrays.asList("15/02/2016 10:15:00 +0000,1,paint1,11",
+ "15/02/2016 10:16:00 +0000,2,paint2,12",
+ "15/02/2016 10:17:00 +0000,3,paint3,13", "15/02/2016 10:18:00 +0000,4,paint4,14",
+ "15/02/2016 10:19:00 +0000,5,paint5,15", "15/02/2016 10:10:00 +0000,6,abcde6,16"));
+
+ Assert.assertTrue(PureStyleSQLApplicationTest.waitTillFileIsPopulated(outputFolder, 40000));
+ lc.shutdown();
+
+ File file = new File(outputFolder);
+ File file1 = new File(outputFolder + file.list()[0]);
+ List<String> strings = FileUtils.readLines(file1);
+
+ String[] actualLines = strings.toArray(new String[strings.size()]);
+ String[] expectedLines = new String[] {
+ "15/02/2016 10:18:00 +0000,15/02/2016 12:00:00 +0000,OILPAINT4",
+ "",
+ "15/02/2016 10:19:00 +0000,15/02/2016 12:00:00 +0000,OILPAINT5",
+ ""};
+ Assert.assertEquals(expectedLines.length, actualLines.length);
+ for (int i = 0; i < actualLines.length; i++) {
+ Assert.assertEquals(expectedLines[i], actualLines[i]);
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/sql/src/test/java/org/apache/apex/malhar/sql/sample/PureStyleSQLApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/sql/src/test/java/org/apache/apex/malhar/sql/sample/PureStyleSQLApplicationTest.java b/examples/sql/src/test/java/org/apache/apex/malhar/sql/sample/PureStyleSQLApplicationTest.java
new file mode 100644
index 0000000..f298059
--- /dev/null
+++ b/examples/sql/src/test/java/org/apache/apex/malhar/sql/sample/PureStyleSQLApplicationTest.java
@@ -0,0 +1,155 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.malhar.sql.sample;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.TimeZone;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+
+import org.apache.apex.malhar.kafka.EmbeddedKafka;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+import com.google.common.collect.Lists;
+
+import com.datatorrent.api.LocalMode;
+
+
+public class PureStyleSQLApplicationTest
+{
+ private final String testTopicData = "dataTopic";
+ private final String testTopicResult = "resultTopic";
+
+ private TimeZone defaultTZ;
+ private EmbeddedKafka kafka;
+ private static String outputFolder = "target/output/";
+
+ @Rule
+ public TestName testName = new TestName();
+
+ @Before
+ public void setUp() throws Exception
+ {
+ defaultTZ = TimeZone.getDefault();
+ TimeZone.setDefault(TimeZone.getTimeZone("GMT"));
+
+ kafka = new EmbeddedKafka();
+ kafka.start();
+ kafka.createTopic(testTopicData);
+ kafka.createTopic(testTopicResult);
+
+ outputFolder += testName.getMethodName() + "/";
+ }
+
+ @After
+ public void tearDown() throws Exception
+ {
+ kafka.stop();
+ TimeZone.setDefault(defaultTZ);
+ }
+
+ @Test
+ public void test() throws Exception
+ {
+ LocalMode lma = LocalMode.newInstance();
+ Configuration conf = new Configuration(false);
+ conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
+ conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties-PureStyleSQLApplication.xml"));
+
+ conf.set("broker", kafka.getBroker());
+ conf.set("topic", testTopicData);
+ conf.set("outputFolder", outputFolder);
+ conf.set("destFileName", "out.tmp");
+
+ PureStyleSQLApplication app = new PureStyleSQLApplication();
+
+ lma.prepareDAG(app, conf);
+
+ LocalMode.Controller lc = lma.getController();
+
+ lc.runAsync();
+ kafka.publish(testTopicData, Arrays.asList(
+ "15/02/2016 10:15:00 +0000,1,paint1,11",
+ "15/02/2016 10:16:00 +0000,2,paint2,12",
+ "15/02/2016 10:17:00 +0000,3,paint3,13",
+ "15/02/2016 10:18:00 +0000,4,paint4,14",
+ "15/02/2016 10:19:00 +0000,5,paint5,15",
+ "15/02/2016 10:10:00 +0000,6,abcde6,16"));
+
+ Assert.assertTrue(waitTillFileIsPopulated(outputFolder, 40000));
+ lc.shutdown();
+
+ File file = new File(outputFolder);
+ File file1 = new File(outputFolder + file.list()[0]);
+ List<String> strings = FileUtils.readLines(file1);
+
+ String[] actualLines = strings.toArray(new String[strings.size()]);
+
+ String[] expectedLines = new String[]{
+ "15/02/2016 10:18:00 +0000,15/02/2016 12:00:00 +0000,OILPAINT4",
+ "",
+ "15/02/2016 10:19:00 +0000,15/02/2016 12:00:00 +0000,OILPAINT5",
+ ""};
+
+ Assert.assertEquals(expectedLines.length, actualLines.length);
+ for (int i = 0;i < expectedLines.length; i++) {
+ Assert.assertEquals(expectedLines[i], actualLines[i]);
+ }
+ }
+
+ public static boolean waitTillFileIsPopulated(String outputFolder, int timeout) throws IOException, InterruptedException
+ {
+ boolean result;
+ long now = System.currentTimeMillis();
+ Path outDir = new Path("file://" + new File(outputFolder).getAbsolutePath());
+ try (FileSystem fs = FileSystem.newInstance(outDir.toUri(), new Configuration())) {
+ List<String> strings = Lists.newArrayList();
+ while (System.currentTimeMillis() - now < timeout) {
+ if (fs.exists(outDir)) {
+ File file = new File(outputFolder);
+ if (file.list().length > 0) {
+ File file1 = new File(outputFolder + file.list()[0]);
+ strings = FileUtils.readLines(file1);
+ if (strings.size() != 0) {
+ break;
+ }
+ }
+ }
+
+ Thread.sleep(500);
+ }
+
+ result = fs.exists(outDir) && (strings.size() != 0);
+ }
+
+ return result;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/sql/src/test/java/org/apache/apex/malhar/sql/sample/SQLApplicationWithAPITest.java
----------------------------------------------------------------------
diff --git a/examples/sql/src/test/java/org/apache/apex/malhar/sql/sample/SQLApplicationWithAPITest.java b/examples/sql/src/test/java/org/apache/apex/malhar/sql/sample/SQLApplicationWithAPITest.java
new file mode 100644
index 0000000..6b1a404
--- /dev/null
+++ b/examples/sql/src/test/java/org/apache/apex/malhar/sql/sample/SQLApplicationWithAPITest.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.malhar.sql.sample;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.TimeZone;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.google.common.base.Predicates;
+import com.google.common.collect.Collections2;
+
+import com.datatorrent.api.LocalMode;
+
+
+public class SQLApplicationWithAPITest
+{
+ private TimeZone defaultTZ;
+
+ @Before
+ public void setUp() throws Exception
+ {
+ defaultTZ = TimeZone.getDefault();
+ TimeZone.setDefault(TimeZone.getTimeZone("GMT"));
+ }
+
+ @After
+ public void tearDown() throws Exception
+ {
+ TimeZone.setDefault(defaultTZ);
+ }
+
+ @Test
+ public void test() throws Exception
+ {
+ LocalMode lma = LocalMode.newInstance();
+ Configuration conf = new Configuration(false);
+ conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
+ conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties-SQLApplicationWithAPI.xml"));
+
+ SQLApplicationWithAPI app = new SQLApplicationWithAPI();
+
+ lma.prepareDAG(app, conf);
+
+ LocalMode.Controller lc = lma.getController();
+
+ PrintStream originalSysout = System.out;
+ final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ System.setOut(new PrintStream(baos));
+
+ lc.runAsync();
+ SQLApplicationWithModelFileTest.waitTillStdoutIsPopulated(baos, 30000);
+ lc.shutdown();
+
+ System.setOut(originalSysout);
+
+ String[] sout = baos.toString().split(System.lineSeparator());
+ Collection<String> filter = Collections2.filter(Arrays.asList(sout), Predicates.containsPattern("Delta Record:"));
+
+ String[] actualLines = filter.toArray(new String[filter.size()]);
+ Assert.assertTrue(actualLines[0].contains("RowTime=Mon Feb 15 10:15:00 GMT 2016, Product=paint1"));
+ Assert.assertTrue(actualLines[1].contains("RowTime=Mon Feb 15 10:16:00 GMT 2016, Product=paint2"));
+ Assert.assertTrue(actualLines[2].contains("RowTime=Mon Feb 15 10:17:00 GMT 2016, Product=paint3"));
+ Assert.assertTrue(actualLines[3].contains("RowTime=Mon Feb 15 10:18:00 GMT 2016, Product=paint4"));
+ Assert.assertTrue(actualLines[4].contains("RowTime=Mon Feb 15 10:19:00 GMT 2016, Product=paint5"));
+ Assert.assertTrue(actualLines[5].contains("RowTime=Mon Feb 15 10:10:00 GMT 2016, Product=abcde6"));
+ }
+}
[16/30] apex-malhar git commit: Renamed demos to examples. Packages
and artifactid names are changed as suggested.
Posted by th...@apache.org.
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/wordcount/src/main/java/com/datatorrent/demos/wordcount/FileWordCount.java
----------------------------------------------------------------------
diff --git a/demos/wordcount/src/main/java/com/datatorrent/demos/wordcount/FileWordCount.java b/demos/wordcount/src/main/java/com/datatorrent/demos/wordcount/FileWordCount.java
deleted file mode 100644
index e8a91b2..0000000
--- a/demos/wordcount/src/main/java/com/datatorrent/demos/wordcount/FileWordCount.java
+++ /dev/null
@@ -1,349 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.wordcount;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.datatorrent.api.Context.OperatorContext;
-import com.datatorrent.api.DefaultInputPort;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.api.annotation.InputPortFieldAnnotation;
-import com.datatorrent.api.annotation.OutputPortFieldAnnotation;
-import com.datatorrent.common.util.BaseOperator;
-
-/**
- * Computes word frequencies per file and globally, and writes the top N pairs to an output file
- * and to snapshot servers for visualization.
- * Currently designed to work with only 1 file at a time; will be enhanced later to support
- * multiple files dropped into the monitored directory at the same time.
- *
- * <p>
- * Receives per-window list of pairs (word, frequency) on the input port. When the end of a file
- * is reached, expects to get an EOF on the control port; at the next endWindow, the top N words
- * and frequencies are computed and emitted to the output ports.
- * <p>
- * There are 3 output ports: (a) One for the per-file top N counts emitted when the file is fully
- * read and is written to the output file. (b) One for the top N counts emitted per window for the
- * current file to the snapshot server and (c) One for the global top N counts emitted per window
- * to a different snapshot server.
- *
- * Since the EOF is received by a single operator, this operator cannot be partitionable
- *
- * @since 3.2.0
- */
-public class FileWordCount extends BaseOperator
-{
- private static final Logger LOG = LoggerFactory.getLogger(FileWordCount.class);
- private static final String GLOBAL = "global";
-
- /**
- * If {@literal topN > 0}, only data for the topN most frequent words is output; if topN == 0, the
- * entire frequency map is output
- */
- protected int topN;
-
- /**
- * Set to true when an EOF control tuple for the current input file is received; reset to false
- * when the corresponding output file has been written.
- */
- protected boolean eof = false;
-
- /**
- * Last component of path (just the file name)
- * incoming value from control tuple
- */
- protected String fileName;
-
- /**
- * {@literal (word => frequency)} map: current file, all words
- */
- protected Map<String, WCPair> wordMapFile = new HashMap<>();
-
- /**
- * {@literal (word => frequency)} map: global, all words
- */
- protected Map<String, WCPair> wordMapGlobal = new HashMap<>();
-
- /**
- * Singleton list with per file data; sent on {@code outputPerFile}
- */
- protected transient List<Map<String, Object>> resultPerFile;
-
- /**
- * Singleton list with global data; sent on {@code outputGlobal}
- */
- protected transient List<Map<String, Object>> resultGlobal;
-
- /**
- * Singleton map of {@code fileName} to sorted list of (word, frequency) pairs
- */
- protected transient Map<String, Object> resultFileFinal;
-
- /**
- * final list of (word, frequency) pairs written to output file
- */
- protected transient List<WCPair> fileFinalList;
-
- /**
- * Input port on which per-window {@literal (word => frequency)} map is received; the map
- * is merged into {@code wordMapFile} and {@code wordMapGlobal}.
- */
- public final transient DefaultInputPort<List<WCPair>> input = new DefaultInputPort<List<WCPair>>()
- {
- @Override
- public void process(List<WCPair> list)
- {
- // blend incoming list into wordMapFile and wordMapGlobal
- for (WCPair pair : list) {
- final String word = pair.word;
- WCPair filePair = wordMapFile.get(word);
- if (null != filePair) { // word seen previously in current file
- WCPair globalPair = wordMapGlobal.get(word); // cannot be null
- filePair.freq += pair.freq;
- globalPair.freq += pair.freq;
- continue;
- }
-
- // new word in current file
- filePair = new WCPair(word, pair.freq);
- wordMapFile.put(word, filePair);
-
- // check global map
- WCPair globalPair = wordMapGlobal.get(word); // may be null
- if (null != globalPair) { // word seen previously
- globalPair.freq += pair.freq;
- continue;
- }
-
- // word never seen before
- globalPair = new WCPair(word, pair.freq);
- wordMapGlobal.put(word, globalPair);
- }
- }
- };
-
- /**
- * Control port on which the current file name is received to indicate EOF
- */
- @InputPortFieldAnnotation(optional = true)
- public final transient DefaultInputPort<String> control = new DefaultInputPort<String>()
- {
- @Override
- public void process(String msg)
- {
- if (msg.isEmpty()) { // sanity check
- throw new RuntimeException("Empty file path");
- }
- LOG.info("FileWordCount: EOF for {}, topN = {}", msg, topN);
- fileName = msg;
- eof = true;
- // NOTE: current version only supports processing one file at a time.
- }
- };
-
- /**
- * Output port for current file output
- */
- public final transient DefaultOutputPort<List<Map<String, Object>>>
- outputPerFile = new DefaultOutputPort<>();
-
- /**
- * Output port for global output
- */
- @OutputPortFieldAnnotation(optional = true)
- public final transient DefaultOutputPort<List<Map<String, Object>>>
- outputGlobal = new DefaultOutputPort<>();
-
- /**
- * Tuple is singleton map {@code fileName => TopNMap} where {@code TopNMap} is the final
- * top N pairs for current file and will be written to the output file; emitted in the
- * {@code endWindow()} call after an EOF
- */
- public final transient DefaultOutputPort<Map<String, Object>>
- fileOutput = new DefaultOutputPort<>();
-
- /**
- * Get the number of top (word, frequency) pairs that will be output
- */
- public int getTopN()
- {
- return topN;
- }
-
- /**
- * Set the number of top (word, frequency) pairs that will be output
- * @param n The new number
- */
- public void setTopN(int n)
- {
- topN = n;
- }
-
- /**
- * {@inheritDoc}
- * Initialize various map and list fields
- */
- @Override
- public void setup(OperatorContext context)
- {
- if (null == wordMapFile) {
- wordMapFile = new HashMap<>();
- }
- if (null == wordMapGlobal) {
- wordMapGlobal = new HashMap<>();
- }
- resultPerFile = new ArrayList(1);
- resultGlobal = new ArrayList(1);
- // singleton map {<fileName> => fileFinalList}; cannot populate it yet since we need fileName
- resultFileFinal = new HashMap<>(1);
- fileFinalList = new ArrayList<>();
- }
-
- /**
- * {@inheritDoc}
- * This is where we do most of the work:
- * 1. Sort global map and emit top N pairs
- * 2. Sort current file map and emit top N pairs
- * 3. If we've seen EOF, emit top N pairs on port connected to file writer and clear all per-file
- * data structures.
- */
- @Override
- public void endWindow()
- {
- LOG.info("FileWordCount: endWindow for {}, topN = {}", fileName, topN);
-
- if (wordMapFile.isEmpty()) { // no words found
- if (eof) { // write empty list to fileOutput port
- // got EOF, so output empty list to output file
- fileFinalList.clear();
- resultFileFinal.put(fileName, fileFinalList);
- fileOutput.emit(resultFileFinal);
-
- // reset for next file
- eof = false;
- fileName = null;
- resultFileFinal.clear();
- }
- LOG.info("FileWordCount: endWindow for {}, no words, topN = {}", fileName, topN);
- return;
- }
-
- LOG.info("FileWordCount: endWindow for {}, wordMapFile.size = {}, topN = {}", fileName, wordMapFile.size(), topN);
-
- // get topN list for this file and, if we have EOF, emit to fileOutput port
-
- // get topN global list and emit to global output port
- getTopNMap(wordMapGlobal, resultGlobal);
- LOG.info("FileWordCount: resultGlobal.size = {}", resultGlobal.size());
- outputGlobal.emit(resultGlobal);
-
- // get topN list for this file and emit to file output port
- getTopNMap(wordMapFile, resultPerFile);
- LOG.info("FileWordCount: resultPerFile.size = {}", resultPerFile.size());
- outputPerFile.emit(resultPerFile);
-
- if (eof) { // got EOF earlier
- if (null == fileName) { // need file name to emit topN pairs to file writer
- throw new RuntimeException("EOF but no fileName at endWindow");
- }
-
- // so compute final topN list from wordMapFile into fileFinalList and emit it
- getTopNList(wordMapFile);
- resultFileFinal.put(fileName, fileFinalList);
- fileOutput.emit(resultFileFinal);
-
- // reset for next file
- eof = false;
- fileName = null;
- wordMapFile.clear();
- resultFileFinal.clear();
- }
- }
-
- /**
- * Get topN frequencies from map, convert each pair to a singleton map and append to result
- * This map is suitable input to AppDataSnapshotServer
- * MUST have {@code map.size() > 0} here
- */
- private void getTopNMap(final Map<String, WCPair> map, List<Map<String, Object>> result)
- {
- final ArrayList<WCPair> list = new ArrayList<>(map.values());
-
- // sort entries in descending order of frequency
- Collections.sort(list, new Comparator<WCPair>()
- {
- @Override
- public int compare(WCPair o1, WCPair o2)
- {
- return (int)(o2.freq - o1.freq);
- }
- });
-
- if (topN > 0) {
- list.subList(topN, map.size()).clear(); // retain only the first topN entries
- }
-
- // convert each pair (word, freq) of list to a map with 2 elements
- // {("word": <word>, "count": freq)} and append to list
- //
- result.clear();
- for (WCPair pair : list) {
- Map<String, Object> wmap = new HashMap<>(2);
- wmap.put("word", pair.word);
- wmap.put("count", pair.freq);
- result.add(wmap);
- }
- LOG.info("FileWordCount:getTopNMap: result.size = {}", result.size());
- list.clear();
- }
-
- /**
- * Populate fileFinalList with topN frequencies from argument
- * This list is suitable input to WordCountWriter which writes it to a file
- * MUST have {@code map.size() > 0} here
- */
- private void getTopNList(final Map<String, WCPair> map)
- {
- fileFinalList.clear();
- fileFinalList.addAll(map.values());
-
- // sort entries in descending order of frequency
- Collections.sort(fileFinalList, new Comparator<WCPair>()
- {
- @Override
- public int compare(WCPair o1, WCPair o2)
- {
- return (int)(o2.freq - o1.freq);
- }
- });
-
- if (topN > 0) {
- fileFinalList.subList(topN, map.size()).clear(); // retain only the first topN entries
- }
- LOG.info("FileWordCount:getTopNList: fileFinalList.size = {}", fileFinalList.size());
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/wordcount/src/main/java/com/datatorrent/demos/wordcount/LineReader.java
----------------------------------------------------------------------
diff --git a/demos/wordcount/src/main/java/com/datatorrent/demos/wordcount/LineReader.java b/demos/wordcount/src/main/java/com/datatorrent/demos/wordcount/LineReader.java
deleted file mode 100644
index 8a1a57b..0000000
--- a/demos/wordcount/src/main/java/com/datatorrent/demos/wordcount/LineReader.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.wordcount;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.fs.Path;
-
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.api.annotation.OutputPortFieldAnnotation;
-import com.datatorrent.lib.io.fs.AbstractFileInputOperator;
-
-/**
- * Reads lines from input file and returns them. If EOF is reached, a control tuple
- * is emitted on the control port
- *
- * @since 3.2.0
- */
-public class LineReader extends AbstractFileInputOperator<String>
-{
- private static final Logger LOG = LoggerFactory.getLogger(LineReader.class);
-
- /**
- * Output port on which lines from current file name are emitted
- */
- public final transient DefaultOutputPort<String> output = new DefaultOutputPort<>();
-
- /**
- * Control port on which the current file name is emitted to indicate EOF
- */
- @OutputPortFieldAnnotation(optional = true)
- public final transient DefaultOutputPort<String> control = new DefaultOutputPort<>();
-
- private transient BufferedReader br = null;
-
- private Path path;
-
- /**
- * File open callback; wrap the file input stream in a buffered reader for reading lines
- * @param curPath The path to the file just opened
- */
- @Override
- protected InputStream openFile(Path curPath) throws IOException
- {
- LOG.info("openFile: curPath = {}", curPath);
- path = curPath;
- InputStream is = super.openFile(path);
- br = new BufferedReader(new InputStreamReader(is));
- return is;
- }
-
- /**
- * File close callback; close buffered reader
- * @param is File input stream that will imminently be closed
- */
- @Override
- protected void closeFile(InputStream is) throws IOException
- {
- super.closeFile(is);
- br.close();
- br = null;
- path = null;
- }
-
- /**
- * {@inheritDoc}
- * If we hit EOF, emit file name on control port
- */
- @Override
- protected String readEntity() throws IOException
- {
- // try to read a line
- final String line = br.readLine();
- if (null != line) { // common case
- LOG.debug("readEntity: line = {}", line);
- return line;
- }
-
- // end-of-file; send control tuple, containing only the last component of the path
- // (only file name) on control port
- //
- if (control.isConnected()) {
- LOG.info("readEntity: EOF for {}", path);
- final String name = path.getName(); // final component of path
- control.emit(name);
- }
-
- return null;
- }
-
- @Override
- protected void emit(String tuple)
- {
- output.emit(tuple);
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/wordcount/src/main/java/com/datatorrent/demos/wordcount/WCPair.java
----------------------------------------------------------------------
diff --git a/demos/wordcount/src/main/java/com/datatorrent/demos/wordcount/WCPair.java b/demos/wordcount/src/main/java/com/datatorrent/demos/wordcount/WCPair.java
deleted file mode 100644
index bb67622..0000000
--- a/demos/wordcount/src/main/java/com/datatorrent/demos/wordcount/WCPair.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.wordcount;
-
-/**
- * A single (word, frequency) pair
- *
- * @since 3.2.0
- */
-public class WCPair
-{
- /**
- * The word
- */
- public String word;
-
- /**
- * The frequency
- */
- public int freq;
-
- /**
- * Default constructor
- */
- public WCPair()
- {
-
- }
-
- /**
- * Create new object with given values
- * @param w The word
- * @param f The frequency
- */
- public WCPair(String w, int f)
- {
- word = w;
- freq = f;
- }
-
- @Override
- public String toString()
- {
- return String.format("(%s, %d)", word, freq);
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/wordcount/src/main/java/com/datatorrent/demos/wordcount/WindowWordCount.java
----------------------------------------------------------------------
diff --git a/demos/wordcount/src/main/java/com/datatorrent/demos/wordcount/WindowWordCount.java b/demos/wordcount/src/main/java/com/datatorrent/demos/wordcount/WindowWordCount.java
deleted file mode 100644
index 0edfd1e..0000000
--- a/demos/wordcount/src/main/java/com/datatorrent/demos/wordcount/WindowWordCount.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.wordcount;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.datatorrent.api.DefaultInputPort;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.common.util.BaseOperator;
-
-/**
- * Computes word frequencies per window and emits them at each {@code endWindow()}. The output is a
- * list of (word, frequency) pairs
- *
- * @since 3.2.0
- */
-public class WindowWordCount extends BaseOperator
-{
- private static final Logger LOG = LoggerFactory.getLogger(WindowWordCount.class);
-
- /** {@literal (word => frequency)} map for current window */
- protected Map<String, WCPair> wordMap = new HashMap<>();
-
- /**
- * Input port on which words are received
- */
- public final transient DefaultInputPort<String> input = new DefaultInputPort<String>()
- {
- @Override
- public void process(String word)
- {
- WCPair pair = wordMap.get(word);
- if (null != pair) { // word seen previously
- pair.freq += 1;
- return;
- }
-
- // new word
- pair = new WCPair();
- pair.word = word;
- pair.freq = 1;
- wordMap.put(word, pair);
- }
- };
-
- /**
- * Output port which emits the list of word frequencies for current window
- */
- public final transient DefaultOutputPort<List<WCPair>> output = new DefaultOutputPort<>();
-
- /**
- * {@inheritDoc}
- * If we've seen some words in this window, emit the map and clear it for next window
- */
- @Override
- public void endWindow()
- {
- LOG.info("WindowWordCount: endWindow");
-
- // got EOF; if no words found, do nothing
- if (wordMap.isEmpty()) {
- return;
- }
-
- // have some words; emit single map and reset for next file
- final ArrayList<WCPair> list = new ArrayList<>(wordMap.values());
- output.emit(list);
- list.clear();
- wordMap.clear();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/wordcount/src/main/java/com/datatorrent/demos/wordcount/WordCountInputOperator.java
----------------------------------------------------------------------
diff --git a/demos/wordcount/src/main/java/com/datatorrent/demos/wordcount/WordCountInputOperator.java b/demos/wordcount/src/main/java/com/datatorrent/demos/wordcount/WordCountInputOperator.java
deleted file mode 100644
index 3a88bab..0000000
--- a/demos/wordcount/src/main/java/com/datatorrent/demos/wordcount/WordCountInputOperator.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.wordcount;
-
-import java.io.BufferedReader;
-import java.io.DataInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.datatorrent.lib.io.SimpleSinglePortInputOperator;
-
-/**
- * <p>WordCountInputOperator class.</p>
- *
- * @since 0.3.2
- */
-public class WordCountInputOperator extends SimpleSinglePortInputOperator<String> implements Runnable
-{
-
- private static final Logger logger = LoggerFactory.getLogger(WordCountInputOperator.class);
- protected long averageSleep = 300;
- protected long sleepPlusMinus = 100;
- protected String fileName = "com/datatorrent/demos/wordcount/samplefile.txt";
-
- public void setAverageSleep(long as)
- {
- averageSleep = as;
- }
-
- public void setSleepPlusMinus(long spm)
- {
- sleepPlusMinus = spm;
- }
-
- public void setFileName(String fn)
- {
- fileName = fn;
- }
-
- @Override
- public void run()
- {
- BufferedReader br = null;
- DataInputStream in = null;
- InputStream fstream = null;
-
- while (true) {
- try {
- String line;
- fstream = this.getClass().getClassLoader().getResourceAsStream(fileName);
-
- in = new DataInputStream(fstream);
- br = new BufferedReader(new InputStreamReader(in));
-
- while ((line = br.readLine()) != null) {
- String[] words = line.trim().split("[\\p{Punct}\\s\\\"\\'\u201c\u201d]+");
- for (String word : words) {
- word = word.trim().toLowerCase();
- if (!word.isEmpty()) {
- outputPort.emit(word);
- }
- }
- try {
- Thread.sleep(averageSleep + (new Double(sleepPlusMinus * (Math.random() * 2 - 1))).longValue());
- } catch (InterruptedException ex) {
- // nothing
- }
- }
-
- } catch (IOException ex) {
- logger.debug(ex.toString());
- } finally {
- try {
- if (br != null) {
- br.close();
- }
- if (in != null) {
- in.close();
- }
- if (fstream != null) {
- fstream.close();
- }
- } catch (IOException exc) {
- // nothing
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/wordcount/src/main/java/com/datatorrent/demos/wordcount/WordCountWriter.java
----------------------------------------------------------------------
diff --git a/demos/wordcount/src/main/java/com/datatorrent/demos/wordcount/WordCountWriter.java b/demos/wordcount/src/main/java/com/datatorrent/demos/wordcount/WordCountWriter.java
deleted file mode 100644
index 30aab10..0000000
--- a/demos/wordcount/src/main/java/com/datatorrent/demos/wordcount/WordCountWriter.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.wordcount;
-
-import java.io.UnsupportedEncodingException;
-
-import java.util.List;
-import java.util.Map;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.datatorrent.lib.io.fs.AbstractFileOutputOperator;
-
-/**
- * Write top N words and their frequencies to a file
- *
- * @since 3.2.0
- */
-public class WordCountWriter extends AbstractFileOutputOperator<Map<String, Object>>
-{
- private static final Logger LOG = LoggerFactory.getLogger(WordCountWriter.class);
- private static final String charsetName = "UTF-8";
- private static final String nl = System.lineSeparator();
-
- private String fileName; // current file name
- private final transient StringBuilder sb = new StringBuilder();
-
- /**
- * {@inheritDoc}
- * Invoke requestFinalize() to create the output file with the desired name without decorations.
- */
- @Override
- public void endWindow()
- {
- if (null != fileName) {
- requestFinalize(fileName);
- }
- super.endWindow();
- }
-
- /**
- * Extracts file name from argument
- * @param tuple Singleton map {@literal (fileName => L) where L is a list of (word, frequency) pairs}
- * @return the file name to write the tuple to
- */
- @Override
- protected String getFileName(Map<String, Object> tuple)
- {
- LOG.info("getFileName: tuple.size = {}", tuple.size());
-
- final Map.Entry<String, Object> entry = tuple.entrySet().iterator().next();
- fileName = entry.getKey();
- LOG.info("getFileName: fileName = {}", fileName);
- return fileName;
- }
-
- /**
- * Extracts output file content from argument
- * @param tuple Singleton map {@literal (fileName => L) where L is a list of (word, frequency) pairs}
- * @return input tuple converted to an array of bytes
- */
- @Override
- protected byte[] getBytesForTuple(Map<String, Object> tuple)
- {
- LOG.info("getBytesForTuple: tuple.size = {}", tuple.size());
-
- // get first and only pair; key is the fileName and is ignored here
- final Map.Entry<String, Object> entry = tuple.entrySet().iterator().next();
- final List<WCPair> list = (List<WCPair>)entry.getValue();
-
- if (sb.length() > 0) { // clear buffer
- sb.delete(0, sb.length());
- }
-
- for ( WCPair pair : list ) {
- sb.append(pair.word);
- sb.append(" : ");
- sb.append(pair.freq);
- sb.append(nl);
- }
-
- final String data = sb.toString();
- LOG.info("getBytesForTuple: data = {}", data);
- try {
- final byte[] result = data.getBytes(charsetName);
- return result;
- } catch (UnsupportedEncodingException ex) {
- throw new RuntimeException("Should never get here", ex);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/wordcount/src/main/java/com/datatorrent/demos/wordcount/WordReader.java
----------------------------------------------------------------------
diff --git a/demos/wordcount/src/main/java/com/datatorrent/demos/wordcount/WordReader.java b/demos/wordcount/src/main/java/com/datatorrent/demos/wordcount/WordReader.java
deleted file mode 100644
index 58c44b4..0000000
--- a/demos/wordcount/src/main/java/com/datatorrent/demos/wordcount/WordReader.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.wordcount;
-
-import java.util.regex.Pattern;
-
-import com.datatorrent.api.Context.OperatorContext;
-import com.datatorrent.api.DefaultInputPort;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.common.util.BaseOperator;
-
-/**
- * Extracts words from input line
- *
- * @since 3.3.0
- */
-public class WordReader extends BaseOperator
-{
- // default pattern for word-separators
- private static final Pattern nonWordDefault = Pattern.compile("[\\p{Punct}\\s]+");
-
- private String nonWordStr; // configurable regex
- private transient Pattern nonWord; // compiled regex
-
- /**
- * Output port on which words from the current file are emitted
- */
- public final transient DefaultOutputPort<String> output = new DefaultOutputPort<>();
-
- /**
- * Input port on which lines from the current file are received
- */
- public final transient DefaultInputPort<String> input = new DefaultInputPort<String>()
- {
-
- @Override
- public void process(String line)
- {
- // line; split it into words and emit them
- final String[] words = nonWord.split(line);
- for (String word : words) {
- if (word.isEmpty()) {
- continue;
- }
- output.emit(word);
- }
- }
- };
-
- /**
- * Returns the regular expression that matches strings between words
- * @return Regular expression for strings that separate words
- */
- public String getNonWordStr()
- {
- return nonWordStr;
- }
-
- /**
- * Sets the regular expression that matches strings between words
- * @param regex New regular expression for strings that separate words
- */
- public void setNonWordStr(String regex)
- {
- nonWordStr = regex;
- }
-
- /**
- * {@inheritDoc}
- * Set nonWord to the default pattern if necessary
- */
- @Override
- public void setup(OperatorContext context)
- {
- if (null == nonWordStr) {
- nonWord = nonWordDefault;
- } else {
- nonWord = Pattern.compile(nonWordStr);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/wordcount/src/main/java/com/datatorrent/demos/wordcount/doc-files/UniqueWordCounter.jpg
----------------------------------------------------------------------
diff --git a/demos/wordcount/src/main/java/com/datatorrent/demos/wordcount/doc-files/UniqueWordCounter.jpg b/demos/wordcount/src/main/java/com/datatorrent/demos/wordcount/doc-files/UniqueWordCounter.jpg
deleted file mode 100644
index 054baed..0000000
Binary files a/demos/wordcount/src/main/java/com/datatorrent/demos/wordcount/doc-files/UniqueWordCounter.jpg and /dev/null differ
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/wordcount/src/main/java/com/datatorrent/demos/wordcount/package-info.java
----------------------------------------------------------------------
diff --git a/demos/wordcount/src/main/java/com/datatorrent/demos/wordcount/package-info.java b/demos/wordcount/src/main/java/com/datatorrent/demos/wordcount/package-info.java
deleted file mode 100644
index d00397d..0000000
--- a/demos/wordcount/src/main/java/com/datatorrent/demos/wordcount/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/**
- * Streaming word count demonstration application.
- */
-package com.datatorrent.demos.wordcount;
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/wordcount/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/demos/wordcount/src/main/resources/META-INF/properties.xml b/demos/wordcount/src/main/resources/META-INF/properties.xml
deleted file mode 100644
index 1d3594e..0000000
--- a/demos/wordcount/src/main/resources/META-INF/properties.xml
+++ /dev/null
@@ -1,98 +0,0 @@
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<configuration>
- <!-- TopNWordsWithQueries -->
-
- <!-- for debugging -->
- <!--
- <property>
- <name>dt.attr.CONTAINER_JVM_OPTIONS</name>
- <value>-Dlog4j.configuration=my_log4j.properties</value>
- </property>
- -->
-
- <!-- monitored input directory -->
- <property>
- <name>dt.application.TopNWordsWithQueries.operator.lineReader.directory</name>
- <value>/tmp/test/input-dir</value>
- </property>
-
- <!-- regular expression for word separator -->
- <property>
- <name>dt.application.TopNWordsWithQueries.operator.wordReader.nonWordStr</name>
- <value>[\p{Punct}\s]+</value>
- </property>
-
- <!-- output directory for word counts -->
- <property>
- <name>dt.application.TopNWordsWithQueries.operator.wcWriter.filePath</name>
- <value>/tmp/test/output-dir</value>
- </property>
-
- <!-- Top N value -->
- <property>
- <name>dt.application.TopNWordsWithQueries.operator.fileWordCount.topN</name>
- <value>10</value>
- </property>
-
- <!-- topic for queries (current file) -->
- <property>
- <name>dt.application.TopNWordsWithQueries.operator.snapshotServerFile.embeddableQueryInfoProvider.topic</name>
- <value>TopNWordsQueryFile</value>
- </property>
-
- <!-- topic for query results (current file) -->
- <property>
- <name>dt.application.TopNWordsWithQueries.operator.wsResultFile.topic</name>
- <value>TopNWordsQueryFileResult</value>
- </property>
-
- <!-- topic for queries (global) -->
- <property>
- <name>dt.application.TopNWordsWithQueries.operator.snapshotServerGlobal.embeddableQueryInfoProvider.topic</name>
- <value>TopNWordsQueryGlobal</value>
- </property>
-
- <!-- topic for query results (global) -->
- <property>
- <name>dt.application.TopNWordsWithQueries.operator.wsResultGlobal.topic</name>
- <value>TopNWordsQueryGlobalResult</value>
- </property>
-
- <!-- retry count -->
- <property>
- <name>dt.application.TwitterDemo.operator.wsResult.numRetries</name>
- <value>2147483647</value>
- </property>
-
-
- <!-- WordCountDemo -->
- <property>
- <name>dt.application.WordCountDemo.operator.wordinput.fileName</name>
- <value>samplefile.txt</value>
- </property>
- <property>
- <name>dt.application.WordCountDemo.stream.wordinput.count.locality</name>
- <value>CONTAINER_LOCAL</value>
- <description>Specify container locality for the viewtuplecount stream
- </description>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/wordcount/src/main/resources/WordDataSchema.json
----------------------------------------------------------------------
diff --git a/demos/wordcount/src/main/resources/WordDataSchema.json b/demos/wordcount/src/main/resources/WordDataSchema.json
deleted file mode 100644
index 5e8e7c0..0000000
--- a/demos/wordcount/src/main/resources/WordDataSchema.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
- "values": [{"name": "word", "type": "string"},
- {"name": "count", "type": "integer"}]
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/wordcount/src/main/resources/com/datatorrent/demos/wordcount/samplefile.txt
----------------------------------------------------------------------
diff --git a/demos/wordcount/src/main/resources/com/datatorrent/demos/wordcount/samplefile.txt b/demos/wordcount/src/main/resources/com/datatorrent/demos/wordcount/samplefile.txt
deleted file mode 100644
index 83eaaed..0000000
--- a/demos/wordcount/src/main/resources/com/datatorrent/demos/wordcount/samplefile.txt
+++ /dev/null
@@ -1 +0,0 @@
-CONTENT DELETED
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/wordcount/src/main/resources/samplefile.txt
----------------------------------------------------------------------
diff --git a/demos/wordcount/src/main/resources/samplefile.txt b/demos/wordcount/src/main/resources/samplefile.txt
deleted file mode 100644
index 02a5e70..0000000
--- a/demos/wordcount/src/main/resources/samplefile.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-CONTENT DELETED
-
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/wordcount/src/site/conf/my-app-conf1.xml
----------------------------------------------------------------------
diff --git a/demos/wordcount/src/site/conf/my-app-conf1.xml b/demos/wordcount/src/site/conf/my-app-conf1.xml
deleted file mode 100644
index f35873b..0000000
--- a/demos/wordcount/src/site/conf/my-app-conf1.xml
+++ /dev/null
@@ -1,27 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<configuration>
- <property>
- <name>dt.attr.MASTER_MEMORY_MB</name>
- <value>1024</value>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/wordcount/src/test/java/com/datatorrent/demos/wordcount/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/demos/wordcount/src/test/java/com/datatorrent/demos/wordcount/ApplicationTest.java b/demos/wordcount/src/test/java/com/datatorrent/demos/wordcount/ApplicationTest.java
deleted file mode 100644
index 1df0459..0000000
--- a/demos/wordcount/src/test/java/com/datatorrent/demos/wordcount/ApplicationTest.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.wordcount;
-
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.conf.Configuration;
-import com.datatorrent.api.LocalMode;
-
-/**
- *
- */
-public class ApplicationTest
-{
- private final transient Logger LOG = LoggerFactory.getLogger(ApplicationTest.class);
- public ApplicationTest()
- {
- }
-
- @Test
- public void testSomeMethod() throws Exception
- {
- LocalMode lma = LocalMode.newInstance();
- Configuration conf = new Configuration(false);
- conf.addResource("dt-site-wordcount.xml");
- lma.prepareDAG(new Application(), conf);
- LocalMode.Controller lc = lma.getController();
- long start = System.currentTimeMillis();
- lc.run(300000);
- long end = System.currentTimeMillis();
- long time = end - start;
- LOG.debug("Test used " + time + " ms");
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/wordcount/src/test/resources/dt-site-wordcount.xml
----------------------------------------------------------------------
diff --git a/demos/wordcount/src/test/resources/dt-site-wordcount.xml b/demos/wordcount/src/test/resources/dt-site-wordcount.xml
deleted file mode 100644
index a25dac4..0000000
--- a/demos/wordcount/src/test/resources/dt-site-wordcount.xml
+++ /dev/null
@@ -1,37 +0,0 @@
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<configuration>
- <property>
- <name>dt.application.WordCountDemo.class</name>
- <value>com.datatorrent.demos.wordcount.Application</value>
- <description>An alias for the application</description>
- </property>
- <property>
- <name>dt.application.WordCountDemo.operator.wordinput.fileName</name>
- <value>samplefile.txt</value>
- </property>
- <property>
- <name>dt.application.WordCountDemo.stream.wordinput.count.locality</name>
- <value>CONTAINER_LOCAL</value>
- <description>Specify container locality for the viewtuplecount stream
- </description>
- </property>
-</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/wordcount/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/demos/wordcount/src/test/resources/log4j.properties b/demos/wordcount/src/test/resources/log4j.properties
deleted file mode 100644
index cf0d19e..0000000
--- a/demos/wordcount/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,43 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-log4j.rootLogger=DEBUG,CONSOLE
-
-log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
-log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
-log4j.appender.CONSOLE.threshold=${test.log.console.threshold}
-test.log.console.threshold=DEBUG
-
-log4j.appender.RFA=org.apache.log4j.RollingFileAppender
-log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
-log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
-log4j.appender.RFA.File=/tmp/app.log
-
-# to enable, add SYSLOG to rootLogger
-log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender
-log4j.appender.SYSLOG.syslogHost=127.0.0.1
-log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout
-log4j.appender.SYSLOG.layout.conversionPattern=${dt.cid} %-5p [%t] %c{2} %x - %m%n
-log4j.appender.SYSLOG.Facility=LOCAL1
-
-log4j.logger.org=info
-#log4j.logger.org.apache.commons.beanutils=warn
-log4j.logger.com.datatorrent=debug
-log4j.logger.org.apache.apex=debug
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/yahoofinance/pom.xml
----------------------------------------------------------------------
diff --git a/demos/yahoofinance/pom.xml b/demos/yahoofinance/pom.xml
deleted file mode 100644
index 819a475..0000000
--- a/demos/yahoofinance/pom.xml
+++ /dev/null
@@ -1,65 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>yahoo-finance-demo</artifactId>
- <packaging>jar</packaging>
-
- <name>Apache Apex Malhar Yahoo! Finance Demo</name>
- <description>Apex demo applications that get Yahoo finance feed and calculate minute price range, minute volume and simple moving average.</description>
-
- <parent>
- <groupId>org.apache.apex</groupId>
- <artifactId>malhar-demos</artifactId>
- <version>3.7.0-SNAPSHOT</version>
- </parent>
-
- <properties>
- <skipTests>true</skipTests>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>net.sf.opencsv</groupId>
- <artifactId>opencsv</artifactId>
- <version>2.0</version>
- </dependency>
- <dependency>
- <groupId>org.apache.derby</groupId>
- <artifactId>derby</artifactId>
- <version>10.9.1.0</version>
- </dependency>
- <dependency>
- <groupId>org.apache.apex</groupId>
- <artifactId>malhar-contrib</artifactId>
- <version>${project.version}</version>
- <exclusions>
- <exclusion>
- <groupId>*</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- </dependencies>
-
-</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/yahoofinance/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/demos/yahoofinance/src/assemble/appPackage.xml b/demos/yahoofinance/src/assemble/appPackage.xml
deleted file mode 100644
index 4138cf2..0000000
--- a/demos/yahoofinance/src/assemble/appPackage.xml
+++ /dev/null
@@ -1,59 +0,0 @@
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
- <id>appPackage</id>
- <formats>
- <format>jar</format>
- </formats>
- <includeBaseDirectory>false</includeBaseDirectory>
- <fileSets>
- <fileSet>
- <directory>${basedir}/target/</directory>
- <outputDirectory>/app</outputDirectory>
- <includes>
- <include>${project.artifactId}-${project.version}.jar</include>
- </includes>
- </fileSet>
- <fileSet>
- <directory>${basedir}/target/deps</directory>
- <outputDirectory>/lib</outputDirectory>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/site/conf</directory>
- <outputDirectory>/conf</outputDirectory>
- <includes>
- <include>*.xml</include>
- </includes>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/main/resources/META-INF</directory>
- <outputDirectory>/META-INF</outputDirectory>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/main/resources/app</directory>
- <outputDirectory>/app</outputDirectory>
- </fileSet>
- </fileSets>
-
-</assembly>
-
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/yahoofinance/src/main/java/com/datatorrent/demos/yahoofinance/ApplicationWithDerbySQL.java
----------------------------------------------------------------------
diff --git a/demos/yahoofinance/src/main/java/com/datatorrent/demos/yahoofinance/ApplicationWithDerbySQL.java b/demos/yahoofinance/src/main/java/com/datatorrent/demos/yahoofinance/ApplicationWithDerbySQL.java
deleted file mode 100644
index 1a38495..0000000
--- a/demos/yahoofinance/src/main/java/com/datatorrent/demos/yahoofinance/ApplicationWithDerbySQL.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.yahoofinance;
-
-import org.apache.apex.malhar.contrib.misc.streamquery.AbstractSqlStreamOperator;
-import org.apache.apex.malhar.contrib.misc.streamquery.DerbySqlStreamOperator;
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-import com.datatorrent.lib.io.ConsoleOutputOperator;
-
-/**
- * This demo will output the stock market data from yahoo finance
- *
- * @since 0.3.2
- */
-@ApplicationAnnotation(name = "YahooFinanceWithDerbySQLDemo")
-public class ApplicationWithDerbySQL implements StreamingApplication
-{
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- String symbolStr = conf.get(ApplicationWithDerbySQL.class.getName() + ".tickerSymbols", "YHOO,GOOG,AAPL,FB,AMZN,NFLX,IBM");
-
- String[] symbols = symbolStr.split(",");
-
- YahooFinanceCSVInputOperator input1 = dag.addOperator("input1", new YahooFinanceCSVInputOperator());
- YahooFinanceCSVInputOperator input2 = dag.addOperator("input2", new YahooFinanceCSVInputOperator());
- DerbySqlStreamOperator sqlOper = dag.addOperator("sqlOper", new DerbySqlStreamOperator());
- ConsoleOutputOperator consoleOperator = dag.addOperator("console", new ConsoleOutputOperator());
-
- for (String symbol : symbols) {
- input1.addSymbol(symbol);
- input2.addSymbol(symbol);
- }
- input1.addFormat("s0");
- input1.addFormat("l1");
- input2.addFormat("s0");
- input2.addFormat("e0");
- input2.addFormat("b4");
-
- AbstractSqlStreamOperator.InputSchema inputSchema1 = new AbstractSqlStreamOperator.InputSchema("t1");
- AbstractSqlStreamOperator.InputSchema inputSchema2 = new AbstractSqlStreamOperator.InputSchema("t2");
- inputSchema1.setColumnInfo("s0", "varchar(100)", true); // symbol
- inputSchema1.setColumnInfo("l1", "float", false); // last trade
- inputSchema2.setColumnInfo("s0", "varchar(100)", true); // symbol
- inputSchema2.setColumnInfo("e0", "float", false); // EPS
- inputSchema2.setColumnInfo("b4", "float", false); // Book value
-
- sqlOper.setInputSchema(0, inputSchema1);
- sqlOper.setInputSchema(1, inputSchema2);
-
- // Calculate PE Ratio and PB Ratio using SQL
- sqlOper.addExecStatementString("SELECT SESSION.t1.s0 AS symbol, SESSION.t1.l1 / SESSION.t2.e0 AS pe_ratio, SESSION.t1.l1 / SESSION.t2.b4 AS pb_ratio FROM SESSION.t1,SESSION.t2 WHERE SESSION.t1.s0 = SESSION.t2.s0");
-
- dag.addStream("input1_sql", input1.outputPort, sqlOper.in1);
- dag.addStream("input2_sql", input2.outputPort, sqlOper.in2);
-
- dag.addStream("result_console", sqlOper.result, consoleOperator.input);
-
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/yahoofinance/src/main/java/com/datatorrent/demos/yahoofinance/StockTickInput.java
----------------------------------------------------------------------
diff --git a/demos/yahoofinance/src/main/java/com/datatorrent/demos/yahoofinance/StockTickInput.java b/demos/yahoofinance/src/main/java/com/datatorrent/demos/yahoofinance/StockTickInput.java
deleted file mode 100644
index 01e3ce9..0000000
--- a/demos/yahoofinance/src/main/java/com/datatorrent/demos/yahoofinance/StockTickInput.java
+++ /dev/null
@@ -1,192 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.yahoofinance;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-
-import javax.validation.constraints.NotNull;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.commons.httpclient.HttpClient;
-import org.apache.commons.httpclient.HttpStatus;
-import org.apache.commons.httpclient.cookie.CookiePolicy;
-import org.apache.commons.httpclient.methods.GetMethod;
-import org.apache.commons.httpclient.params.DefaultHttpParams;
-import org.apache.hadoop.util.StringUtils;
-
-import com.datatorrent.api.Context.OperatorContext;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.api.InputOperator;
-import com.datatorrent.api.annotation.OutputPortFieldAnnotation;
-import com.datatorrent.lib.util.KeyValPair;
-
-import au.com.bytecode.opencsv.CSVReader;
-
-/**
- * This operator sends price, volume and time into separate ports and calculates incremental volume.
- *
- * @since 0.3.2
- */
-public class StockTickInput implements InputOperator
-{
- private static final Logger logger = LoggerFactory.getLogger(StockTickInput.class);
- /**
- * Timeout interval for reading from server. 0 or negative indicates no timeout.
- */
- public int readIntervalMillis = 500;
- /**
- * The URL of the web service resource for the POST request.
- */
- private String url;
- private String[] symbols;
- @NotNull
- private String tickers;
- private transient HttpClient client;
- private transient GetMethod method;
- private HashMap<String, Long> lastVolume = new HashMap<String, Long>();
- private boolean outputEvenIfZeroVolume = false;
- /**
- * The output port to emit price.
- */
- @OutputPortFieldAnnotation(optional = true)
- public final transient DefaultOutputPort<KeyValPair<String, Double>> price = new DefaultOutputPort<KeyValPair<String, Double>>();
- /**
- * The output port to emit incremental volume.
- */
- @OutputPortFieldAnnotation(optional = true)
- public final transient DefaultOutputPort<KeyValPair<String, Long>> volume = new DefaultOutputPort<KeyValPair<String, Long>>();
- /**
- * The output port to emit last traded time.
- */
- @OutputPortFieldAnnotation(optional = true)
- public final transient DefaultOutputPort<KeyValPair<String, String>> time = new DefaultOutputPort<KeyValPair<String, String>>();
-
- /**
- * Prepare URL from symbols and parameters. URL will be something like: http://download.finance.yahoo.com/d/quotes.csv?s=IBM,GOOG,AAPL,YHOO&f=sl1vt1
- *
- * @return the URL
- */
- private String prepareURL()
- {
- String str = "http://download.finance.yahoo.com/d/quotes.csv?s=";
- for (int i = 0; i < symbols.length; i++) {
- if (i != 0) {
- str += ",";
- }
- str += symbols[i];
- }
- str += "&f=sl1vt1&e=.csv";
- return str;
- }
-
- @Override
- public void setup(OperatorContext context)
- {
- url = prepareURL();
- client = new HttpClient();
- method = new GetMethod(url);
- DefaultHttpParams.getDefaultParams().setParameter("http.protocol.cookie-policy", CookiePolicy.BROWSER_COMPATIBILITY);
- }
-
- @Override
- public void teardown()
- {
- }
-
- @Override
- public void emitTuples()
- {
-
- try {
- int statusCode = client.executeMethod(method);
- if (statusCode != HttpStatus.SC_OK) {
- logger.error("Method failed: " + method.getStatusLine());
- } else {
- InputStream istream = method.getResponseBodyAsStream();
- // Process response
- InputStreamReader isr = new InputStreamReader(istream);
- CSVReader reader = new CSVReader(isr);
- List<String[]> myEntries = reader.readAll();
- for (String[] stringArr: myEntries) {
- ArrayList<String> tuple = new ArrayList<String>(Arrays.asList(stringArr));
- if (tuple.size() != 4) {
- return;
- }
- // input csv is <Symbol>,<Price>,<Volume>,<Time>
- String symbol = tuple.get(0);
- double currentPrice = Double.valueOf(tuple.get(1));
- long currentVolume = Long.valueOf(tuple.get(2));
- String timeStamp = tuple.get(3);
- long vol = currentVolume;
- // Sends total volume in first tick, and incremental volume afterwards.
- if (lastVolume.containsKey(symbol)) {
- vol -= lastVolume.get(symbol);
- }
-
- if (vol > 0 || outputEvenIfZeroVolume) {
- price.emit(new KeyValPair<String, Double>(symbol, currentPrice));
- volume.emit(new KeyValPair<String, Long>(symbol, vol));
- time.emit(new KeyValPair<String, String>(symbol, timeStamp));
- lastVolume.put(symbol, currentVolume);
- }
- }
- }
- Thread.sleep(readIntervalMillis);
- } catch (InterruptedException ex) {
- logger.debug(ex.toString());
- } catch (IOException ex) {
- logger.debug(ex.toString());
- }
- }
-
- @Override
- public void beginWindow(long windowId)
- {
- }
-
- @Override
- public void endWindow()
- {
- }
-
- public void setOutputEvenIfZeroVolume(boolean outputEvenIfZeroVolume)
- {
- this.outputEvenIfZeroVolume = outputEvenIfZeroVolume;
- }
-
- public void setTickers(String tickers)
- {
- this.tickers = tickers;
- symbols = StringUtils.split(tickers, ',');
- }
-
- public String getTickers()
- {
- return tickers;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/yahoofinance/src/main/java/com/datatorrent/demos/yahoofinance/YahooFinanceApplication.java
----------------------------------------------------------------------
diff --git a/demos/yahoofinance/src/main/java/com/datatorrent/demos/yahoofinance/YahooFinanceApplication.java b/demos/yahoofinance/src/main/java/com/datatorrent/demos/yahoofinance/YahooFinanceApplication.java
deleted file mode 100644
index a6aaece..0000000
--- a/demos/yahoofinance/src/main/java/com/datatorrent/demos/yahoofinance/YahooFinanceApplication.java
+++ /dev/null
@@ -1,365 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.yahoofinance;
-
-import org.apache.hadoop.conf.Configuration;
-import com.datatorrent.api.Context.OperatorContext;
-import com.datatorrent.api.Context.PortContext;
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.Operator.InputPort;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-import com.datatorrent.lib.io.ConsoleOutputOperator;
-import com.datatorrent.lib.math.RangeKeyVal;
-import com.datatorrent.lib.math.SumKeyVal;
-import com.datatorrent.lib.multiwindow.SimpleMovingAverage;
-import com.datatorrent.lib.stream.ConsolidatorKeyVal;
-import com.datatorrent.lib.util.BaseKeyValueOperator.DefaultPartitionCodec;
-import com.datatorrent.lib.util.HighLow;
-
-/**
- * Yahoo! Finance Application Demo :<br>
- * Get Yahoo finance feed and calculate minute price range, minute volume,
- * simple moving average of 5 minutes. <br>
- * <br>
- * Functional Description : <br>
- * Application samples yahoo finance ticker every 200ms. All data points in one
- * second are streamed from input adapter. <br>
- * <br>
- *
- * Application calculates following Real Time Value(s):<br>
- * <ul>
- * <li>Quotes for IBM, Google, Apple, Yahoo stocks price/volume/time displayed
- * every second.</li>
- * <li>Charts for Stocks in terms for high/low price vs volume for last minute.</li>
- * <li>Simple moving average over last 5 minutes for IBM, Google, Apple, Yahoo
- * stocks.</li>
- * </ul>
- * <br>
- * <br>
- *
- * Custom Attribute : <br>
- * <ul>
- * <li>Application streaming window size(STREAMING_WINDOW_SIZE_MILLIS) = 1 sec,
- * since we are only interested in quotes every second.</li>
- * <li>Range/Minute Volume operator's window size(APPLICATION_WINDOW_COUNT) =
- * 60, aggregate over one minute.</li>
- * <li>Sum operator window length : 300, sliding average over last 5 minutes.</li>
- * </ul>
- * <br>
- *
- * Input Adapter : <br>
- * Stock Tick input operator get yahoo finance real time stock quotes data and
- * pushes application. <br>
- * <br>
- *
- * Output Adapter : <br>
- * Output values are written to console through ConsoleOutputOerator<br>
- * if you need to change write to HDFS,HTTP .. instead of console, <br>
- * Please refer to {@link com.datatorrent.lib.io.HttpOutputOperator} or
- * {@link com.datatorrent.lib.io.fs.HdfsOutputOperator}. <br>
- * <br>
- *
- * Run Sample Application : <br>
- * <p>
- * Running Java Test or Main app in IDE:
- *
- * <pre>
- * LocalMode.runApp(new Application(), 600000); // 10 min run
- * </pre>
- *
- * Run Success : <br>
- * For successful deployment and run, user should see following output on
- * console:
- *
- * <pre>
- * Price SMA: AAPL=435.965
- * Price SMA: GOOG=877.0
- * QUOTE: {YHOO=[26.37, 9760360, 4:00pm, null, null], IBM=[203.77, 2899698, 4:00pm, null, null], GOOG=[877.0, 2069614, 4:00pm, null, null], AAPL=[435.965, 10208099, 4:00pm, null, null]}
- * Price SMA: YHOO=26.37
- * </pre>
- *
- * Scaling Options : <br>
- * <ul>
- * <li>Volume operator can be replicated using a {@link StatelessPartitioner}
- * on an operator.</li>
- * <li>Range value operator can replicated but using proper unifier
- * operator(read App Dev Guide).</li>
- * <li>Slinging window operator can be replicated with proper unifier operator.</li>
- * </ul>
- * <br>
- *
- * Application DAG : <br>
- * <img src="doc-files/Application.gif" width=600px > <br>
- * <br>
- *
- * Streaming Window Size : 1000 ms(1 Sec) <br>
- * Operator Details : <br>
- * <ul>
- * <li>
- * <p>
- * <b>The operator DailyVolume:</b> This operator reads from the input port,
- * which contains the incremental volume tuples from StockTickInput, and
- * aggregates the data to provide the cumulative volume. It just utilizes the
- * library class SumKeyVal<K,V> provided in math package. In this case,
- * SumKeyVal<String,Long>, where K is the stock symbol, V is the aggregated
- * volume, with cumulative set to true. (Otherwise if cumulative was set to
- * false, SumKeyVal would provide the sum for the application window.) The platform
- * provides a number of built-in operators for simple operations like this so
- * that application developers do not have to write them. More examples to
- * follow. This operator assumes that the application restarts before market
- * opens every day.
- * </p>
- * Class : {@link com.datatorrent.lib.math.SumKeyVal} <br>
- * Operator Application Window Count : 1 <br>
- * StateFull : Yes, volume gets aggregated every window count.</li>
- *
- * <li>
- * <p>
- * <b>The operator MinuteVolume:</b> This operator reads from the input port,
- * which contains the volume tuples from StockTickInput, and aggregates the data
- * to provide the sum of the volume within one minute. Like the operator
- * DailyVolume, this operator is also SumKeyVal<String,Long>, but with
- * cumulative set to false. Application Window is set to 1 minute. We will
- * explain how to set this later. <br>
- * Class : {@link com.datatorrent.lib.math.SumKeyVal} <br>
- * Operator App Window Count : 60 (1 Minute) <br>
- * StateFull : Yes, aggregate over last 60 windows.</li>
- *
- * <li>
- * <p>
- * <b>The operator Quote:</b> This operator has three input ports, which are
- * price (from StockTickInput), daily_vol (from Daily Volume), and time (from
- * StockTickInput). This operator just consolidates the three data and and emits
- * the consolidated data. It utilizes the class ConsolidatorKeyVal<K> from
- * stream package.<br>
- * Class : {@link com.datatorrent.lib.stream.ConsolidatorKeyVal} <br>
- * Operator App Window Count : 1 <br>
- * StateFull : No</li>
- *
- * <li>
- * <p>
- * <b>The operator Chart:</b> This operator is very similar to the operator
- * Quote, except that it takes inputs from High Low and Minute Vol and outputs
- * the consolidated tuples to the output port. <br>
- * Class : {@link com.datatorrent.lib.stream.ConsolidatorKeyVal} <br>
- * StateFull : No<br>
- * Operator App Window Count : 1</li>
- *
- *
- * <li>
- * <p>
- * <b>The operator PriceSMA:</b> SMA stands for - Simple Moving Average. It
- * reads from the input port, which contains the price tuples from
- * StockTickInput, and provides the moving average price of the stock. It
- * utilizes SimpleMovingAverage<String,Double>, which is provided in multiwindow
- * package. SimpleMovingAverage keeps track of the data of the previous N
- * application windows in a sliding manner. For each end window event, it
- * provides the average of the data in those application windows. <br>
- * Class : {@link com.datatorrent.lib.multiwindow.SimpleMovingAverage} <br>
- * StateFull : Yes, stores values across application window. <br>
- * Operator App Window : 1 <br>
- * Operator Sliding Window : 300 (5 mins).</li>
- *
- * <li>
- * <p>
- * <b>The operator Console: </b> This operator just outputs the input tuples to
- * the console (or stdout). In this example, there are four console operators,
- * which connect to the output of Quote, Chart, PriceSMA and VolumeSMA. In
- * practice, they should be replaced by operators which would do something about
- * the data, like drawing charts. </li>
- *
- * </ul>
- * <br>
- *
- * @since 0.3.2
- */
-@ApplicationAnnotation(name = "YahooFinanceDemo")
-public class YahooFinanceApplication implements StreamingApplication
-{
- protected int streamingWindowSizeMilliSeconds = 1000; // 1 second
- protected int appWindowCountMinute = 60; // 1 minute
- protected int appWindowCountSMA = 300; // 5 minute
- //protected String[] tickers = {"IBM", "GOOG", "AAPL", "YHOO"};
-
- /**
- * Instantiate stock input operator for actual Yahoo finance ticks of symbol, last price, total daily volume, and last traded price.
- * @param name Operator name
- * @param dag Application DAG graph.
- * @return StockTickInput instance.
- */
- public StockTickInput getStockTickInputOperator(String name, DAG dag)
- {
- StockTickInput oper = dag.addOperator(name, StockTickInput.class);
- oper.readIntervalMillis = 200;
- //oper.symbols = tickers;
- return oper;
- }
-
- /**
- * Instantiate {@link com.datatorrent.lib.math.SumKeyVal} operator
- * to sends total daily volume by adding volumes from each ticks.
- * @param name Operator name
- * @param dag Application DAG graph.
- * @return SumKeyVal instance.
- */
- public SumKeyVal<String, Long> getDailyVolumeOperator(String name, DAG dag)
- {
- SumKeyVal<String, Long> oper = dag.addOperator(name, new SumKeyVal<String, Long>());
- oper.setType(Long.class);
- oper.setCumulative(true);
- return oper;
- }
-
- /**
- * Instantiate {@link com.datatorrent.lib.math.SumKeyVal} operator
- * Get aggregated volume of 1 minute and send at the end window of 1 minute.
- * @param name Operator name
- * @param dag Application DAG graph.
- * @param appWindowCount Operator aggregate window size.
- * @return SumKeyVal instance.
- */
- public SumKeyVal<String, Long> getMinuteVolumeOperator(String name, DAG dag, int appWindowCount)
- {
- SumKeyVal<String, Long> oper = dag.addOperator(name, new SumKeyVal<String, Long>());
- oper.setType(Long.class);
- dag.getOperatorMeta(name).getAttributes().put(OperatorContext.APPLICATION_WINDOW_COUNT, appWindowCount);
- return oper;
- }
-
- /**
- * Instantiate {@link com.datatorrent.lib.math.RangeKeyVal} operator to get high/low
- * value for each key within given application window.
- * Get High-low range for 1 minute.
- * @param name Operator name
- * @param dag Application DAG graph.
- * @param appWindowCount Operator aggregate window size.
- * @return RangeKeyVal instance.
- */
- public RangeKeyVal<String, Double> getHighLowOperator(String name, DAG dag, int appWindowCount)
- {
- RangeKeyVal<String, Double> oper = dag.addOperator(name, new RangeKeyVal<String, Double>());
- dag.getOperatorMeta(name).getAttributes().put(OperatorContext.APPLICATION_WINDOW_COUNT, appWindowCount);
- oper.setType(Double.class);
- return oper;
- }
-
- /**
- * Instantiate {@link com.datatorrent.lib.stream.ConsolidatorKeyVal} to send
- * Quote (Merge price, daily volume, time)
- * @param name Operator name
- * @param dag Application DAG graph.
- * @return ConsolidatorKeyVal instance.
- */
- public ConsolidatorKeyVal<String,Double,Long,String,?,?> getQuoteOperator(String name, DAG dag)
- {
- ConsolidatorKeyVal<String,Double,Long,String,?,?> oper = dag.addOperator(name, new ConsolidatorKeyVal<String,Double,Long,String,Object,Object>());
- return oper;
- }
-
- /**
- * Instantiate {@link com.datatorrent.lib.stream.ConsolidatorKeyVal} to send
- * Chart (Merge minute volume and minute high-low)
- * @param name Operator name
- * @param dag Application DAG graph.
- * @return ConsolidatorKeyVal instance.
- */
- public ConsolidatorKeyVal<String,HighLow<Double>,Long,?,?,?> getChartOperator(String name, DAG dag)
- {
- ConsolidatorKeyVal<String,HighLow<Double>,Long,?,?,?> oper = dag.addOperator(name, new ConsolidatorKeyVal<String,HighLow<Double>,Long,Object,Object,Object>());
- return oper;
- }
-
- /**
- * Instantiate {@link com.datatorrent.lib.multiwindow.SimpleMovingAverage} to calculate moving average for price
- * over given window size. Sliding window size is 1.
- * @param name Operator name
- * @param dag Application DAG graph.
- * @param appWindowCount Operator aggregate window size.
- * @return SimpleMovingAverage instance.
- */
- public SimpleMovingAverage<String, Double> getPriceSimpleMovingAverageOperator(String name, DAG dag, int appWindowCount)
- {
- SimpleMovingAverage<String, Double> oper = dag.addOperator(name, new SimpleMovingAverage<String, Double>());
- oper.setWindowSize(appWindowCount);
- oper.setType(Double.class);
- return oper;
- }
-
- /**
- * Get console for output operator.
- * @param name Operator name
- * @param dag Application DAG graph.
- * @return input port for console output.
- */
- public InputPort<Object> getConsole(String name, /*String nodeName,*/ DAG dag, String prefix)
- {
- // hack to output to HTTP based on actual environment
- /*
- String serverAddr = System.getenv("MALHAR_AJAXSERVER_ADDRESS");
- if (serverAddr != null) {
- HttpOutputOperator<Object> oper = dag.addOperator(name, new HttpOutputOperator<Object>());
- oper.setResourceURL(URI.create("http://" + serverAddr + "/channel/" + nodeName));
- return oper.input;
- }
- */
-
- ConsoleOutputOperator oper = dag.addOperator(name, ConsoleOutputOperator.class);
- oper.setStringFormat(prefix + ": %s");
- return oper.input;
- }
-
- /**
- * Populate Yahoo Finance Demo Application DAG.
- */
- @SuppressWarnings("unchecked")
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
-
- dag.getAttributes().put(DAG.STREAMING_WINDOW_SIZE_MILLIS, streamingWindowSizeMilliSeconds);
-
- StockTickInput tick = getStockTickInputOperator("StockTickInput", dag);
- SumKeyVal<String, Long> dailyVolume = getDailyVolumeOperator("DailyVolume", dag);
- ConsolidatorKeyVal<String,Double,Long,String,?,?> quoteOperator = getQuoteOperator("Quote", dag);
-
- RangeKeyVal<String, Double> highlow = getHighLowOperator("HighLow", dag, appWindowCountMinute);
- SumKeyVal<String, Long> minuteVolume = getMinuteVolumeOperator("MinuteVolume", dag, appWindowCountMinute);
- ConsolidatorKeyVal<String,HighLow<Double>,Long,?,?,?> chartOperator = getChartOperator("Chart", dag);
-
- SimpleMovingAverage<String, Double> priceSMA = getPriceSimpleMovingAverageOperator("PriceSMA", dag, appWindowCountSMA);
- DefaultPartitionCodec<String, Double> codec = new DefaultPartitionCodec<String, Double>();
- dag.setInputPortAttribute(highlow.data, PortContext.STREAM_CODEC, codec);
- dag.setInputPortAttribute(priceSMA.data, PortContext.STREAM_CODEC, codec);
- dag.addStream("price", tick.price, quoteOperator.in1, highlow.data, priceSMA.data);
- dag.addStream("vol", tick.volume, dailyVolume.data, minuteVolume.data);
- dag.addStream("time", tick.time, quoteOperator.in3);
- dag.addStream("daily_vol", dailyVolume.sum, quoteOperator.in2);
-
- dag.addStream("quote_data", quoteOperator.out, getConsole("quoteConsole", dag, "QUOTE"));
-
- dag.addStream("high_low", highlow.range, chartOperator.in1);
- dag.addStream("vol_1min", minuteVolume.sum, chartOperator.in2);
- dag.addStream("chart_data", chartOperator.out, getConsole("chartConsole", dag, "CHART"));
-
- dag.addStream("sma_price", priceSMA.doubleSMA, getConsole("priceSMAConsole", dag, "Price SMA"));
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/yahoofinance/src/main/java/com/datatorrent/demos/yahoofinance/YahooFinanceCSVInputOperator.java
----------------------------------------------------------------------
diff --git a/demos/yahoofinance/src/main/java/com/datatorrent/demos/yahoofinance/YahooFinanceCSVInputOperator.java b/demos/yahoofinance/src/main/java/com/datatorrent/demos/yahoofinance/YahooFinanceCSVInputOperator.java
deleted file mode 100644
index cf3801e..0000000
--- a/demos/yahoofinance/src/main/java/com/datatorrent/demos/yahoofinance/YahooFinanceCSVInputOperator.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.yahoofinance;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.commons.httpclient.HttpClient;
-import org.apache.commons.httpclient.HttpStatus;
-import org.apache.commons.httpclient.cookie.CookiePolicy;
-import org.apache.commons.httpclient.methods.GetMethod;
-import org.apache.commons.httpclient.params.DefaultHttpParams;
-
-import com.datatorrent.api.Context.OperatorContext;
-import com.datatorrent.lib.io.SimpleSinglePortInputOperator;
-
-import au.com.bytecode.opencsv.CSVReader;
-
-/**
- * Grabs Yahoo Finance quotes data and emits HashMap, with key equals the format name (e.g. "s0") <p>
- *
- * @since 0.3.2
- */
-public class YahooFinanceCSVInputOperator extends SimpleSinglePortInputOperator<HashMap<String, Object>> implements Runnable
-{
- private static final Logger logger = LoggerFactory.getLogger(YahooFinanceCSVInputOperator.class);
- /**
- * Timeout interval for reading from server. 0 or negative indicates no timeout.
- */
- private int readIntervalMillis = 500;
-
- /**
- * The URL of the web service resource for the POST request.
- */
- private String url;
- private transient HttpClient client;
- private transient GetMethod method;
-
- private ArrayList<String> symbolList = new ArrayList<String>();
- private ArrayList<String> parameterList = new ArrayList<String>();
-
- public void addSymbol(String symbol)
- {
- symbolList.add(symbol);
- }
-
- public void addFormat(String format)
- {
- parameterList.add(format);
- }
-
- public ArrayList<String> getSymbolList()
- {
- return symbolList;
- }
-
- public ArrayList<String> getParameterList()
- {
- return parameterList;
- }
-
- public int getReadIntervalMillis()
- {
- return readIntervalMillis;
- }
-
- public void setReadIntervalMillis(int readIntervalMillis)
- {
- this.readIntervalMillis = readIntervalMillis;
- }
-
- /**
- * Prepare URL from symbols and parameters.
- * URL will be something like: http://download.finance.yahoo.com/d/quotes.csv?s=GOOG,FB,YHOO&f=sl1vt1&e=.csv
- * @return
- */
- private String prepareURL()
- {
- String str = "http://download.finance.yahoo.com/d/quotes.csv?";
-
- str += "s=";
- for (int i = 0; i < symbolList.size(); i++) {
- if (i != 0) {
- str += ",";
- }
- str += symbolList.get(i);
- }
- str += "&f=";
- for (String format: parameterList) {
- str += format;
- }
- str += "&e=.csv";
- return str;
- }
-
- @Override
- public void setup(OperatorContext context)
- {
- url = prepareURL();
- client = new HttpClient();
- method = new GetMethod(url);
- DefaultHttpParams.getDefaultParams().setParameter("http.protocol.cookie-policy", CookiePolicy.BROWSER_COMPATIBILITY);
- }
-
- @Override
- public void run()
- {
- while (true) {
- try {
- int statusCode = client.executeMethod(method);
- if (statusCode != HttpStatus.SC_OK) {
- logger.error("Method failed: " + method.getStatusLine());
- } else {
- InputStream istream;
- istream = method.getResponseBodyAsStream();
- // Process response
- InputStreamReader isr = new InputStreamReader(istream);
- CSVReader reader = new CSVReader(isr);
- List<String[]> myEntries;
- myEntries = reader.readAll();
- for (String[] stringArr: myEntries) {
- HashMap<String,Object> hm = new HashMap<String,Object>();
- for (int i = 0; i < parameterList.size(); i++) {
- hm.put(parameterList.get(i), stringArr[i]);
- }
- outputPort.emit(hm); // send out one symbol at a time
- }
- }
- Thread.sleep(readIntervalMillis);
- } catch (InterruptedException ex) {
- logger.debug(ex.toString());
- } catch (IOException ex) {
- logger.debug(ex.toString());
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/yahoofinance/src/main/java/com/datatorrent/demos/yahoofinance/doc-files/Application.gif
----------------------------------------------------------------------
diff --git a/demos/yahoofinance/src/main/java/com/datatorrent/demos/yahoofinance/doc-files/Application.gif b/demos/yahoofinance/src/main/java/com/datatorrent/demos/yahoofinance/doc-files/Application.gif
deleted file mode 100644
index 8fc8331..0000000
Binary files a/demos/yahoofinance/src/main/java/com/datatorrent/demos/yahoofinance/doc-files/Application.gif and /dev/null differ
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/yahoofinance/src/main/java/com/datatorrent/demos/yahoofinance/package-info.java
----------------------------------------------------------------------
diff --git a/demos/yahoofinance/src/main/java/com/datatorrent/demos/yahoofinance/package-info.java b/demos/yahoofinance/src/main/java/com/datatorrent/demos/yahoofinance/package-info.java
deleted file mode 100644
index 992bc30..0000000
--- a/demos/yahoofinance/src/main/java/com/datatorrent/demos/yahoofinance/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/**
- * Yahoo Finance demonstration applications.
- */
-package com.datatorrent.demos.yahoofinance;
[27/30] apex-malhar git commit: Renamed demos to examples. Packages
and artifactid names are changed as suggested.
Posted by th...@apache.org.
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/MinimalWordCount.java
----------------------------------------------------------------------
diff --git a/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/MinimalWordCount.java b/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/MinimalWordCount.java
deleted file mode 100644
index 327c882..0000000
--- a/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/MinimalWordCount.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.apex.malhar.stream.sample;
-
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.apex.malhar.lib.window.TriggerOption;
-import org.apache.apex.malhar.lib.window.Tuple;
-import org.apache.apex.malhar.lib.window.WindowOption;
-import org.apache.apex.malhar.stream.api.function.Function;
-import org.apache.apex.malhar.stream.api.impl.StreamFactory;
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.Context;
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.DefaultInputPort;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-import com.datatorrent.common.util.BaseOperator;
-import com.datatorrent.lib.util.KeyValPair;
-
-import static org.apache.apex.malhar.stream.api.Option.Options.name;
-
-/**
- * Beam MinimalWordCount Example
- *
- * @since 3.5.0
- */
-@ApplicationAnnotation(name = "MinimalWordCount")
-public class MinimalWordCount implements StreamingApplication
-{
- public static class Collector extends BaseOperator
- {
- static Map<String, Long> result;
- private static boolean done = false;
-
- public static boolean isDone()
- {
- return done;
- }
-
- @Override
- public void setup(Context.OperatorContext context)
- {
- done = false;
- result = new HashMap<>();
- }
-
- public final transient DefaultInputPort<KeyValPair<String, Long>> input = new DefaultInputPort<KeyValPair<String, Long>>()
- {
- @Override
- public void process(KeyValPair<String, Long> tuple)
- {
- if (tuple.getKey().equals("bye")) {
- done = true;
- }
- result.put(tuple.getKey(), tuple.getValue());
- }
- };
- }
-
- /**
- * Populate the dag using High-Level API.
- * @param dag
- * @param conf
- */
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- Collector collector = new Collector();
- // Create a stream reading from a file line by line using StreamFactory.
- StreamFactory.fromFolder("./src/test/resources/wordcount", name("textInput"))
- // Use a flatmap transformation to extract words from the incoming stream of lines.
- .flatMap(new Function.FlatMapFunction<String, String>()
- {
- @Override
- public Iterable<String> f(String input)
- {
- return Arrays.asList(input.split("[^a-zA-Z']+"));
-
- }
- }, name("ExtractWords"))
- // Apply windowing to the stream for counting, in this case, the window option is global window.
- .window(new WindowOption.GlobalWindow(), new TriggerOption().accumulatingFiredPanes().withEarlyFiringsAtEvery(1))
- // Count the appearances of every word.
- .countByKey(new Function.ToKeyValue<String, String, Long>()
- {
- @Override
- public Tuple<KeyValPair<String, Long>> f(String input)
- {
- return new Tuple.PlainTuple<KeyValPair<String, Long>>(new KeyValPair<String, Long>(input, 1L));
- }
- }, name("countByKey"))
- // Format the counting result to a readable format by unwrapping the tuples.
- .map(new Function.MapFunction<Tuple.WindowedTuple<KeyValPair<String, Long>>, KeyValPair<String, Long>>()
- {
- @Override
- public KeyValPair<String, Long> f(Tuple.WindowedTuple<KeyValPair<String, Long>> input)
- {
- return input.getValue();
- }
- }, name("FormatResults"))
- // Print the result.
- .print(name("console"))
- // Attach a collector to the stream to collect results.
- .endWith(collector, collector.input, name("Collector"))
- // populate the dag using the stream.
- .populateDag(dag);
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/WindowedWordCount.java
----------------------------------------------------------------------
diff --git a/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/WindowedWordCount.java b/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/WindowedWordCount.java
deleted file mode 100644
index 5b83bd0..0000000
--- a/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/WindowedWordCount.java
+++ /dev/null
@@ -1,290 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.apex.malhar.stream.sample;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.joda.time.Duration;
-import org.apache.apex.malhar.lib.window.TriggerOption;
-import org.apache.apex.malhar.lib.window.Tuple;
-import org.apache.apex.malhar.lib.window.WindowOption;
-import org.apache.apex.malhar.stream.api.ApexStream;
-import org.apache.apex.malhar.stream.api.WindowedStream;
-import org.apache.apex.malhar.stream.api.function.Function;
-import org.apache.apex.malhar.stream.api.impl.StreamFactory;
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.conf.Configuration;
-
-import com.google.common.base.Throwables;
-import com.datatorrent.api.Context;
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.DefaultInputPort;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.api.InputOperator;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-import com.datatorrent.common.util.BaseOperator;
-import com.datatorrent.lib.util.KeyValPair;
-
-import static org.apache.apex.malhar.stream.api.Option.Options.name;
-
-/**
- * Beam WindowedWordCount Example.
- *
- * @since 3.5.0
- */
-@ApplicationAnnotation(name = "WindowedWordCount")
-public class WindowedWordCount implements StreamingApplication
-{
- static final int WINDOW_SIZE = 1; // Default window duration in minutes
-
- /**
- * A input operator that reads from and output a file line by line to downstream with a time gap between
- * every two lines.
- */
- public static class TextInput extends BaseOperator implements InputOperator
- {
- public final transient DefaultOutputPort<String> output = new DefaultOutputPort<>();
- private boolean done = false;
-
- private transient BufferedReader reader;
-
- @Override
- public void setup(Context.OperatorContext context)
- {
- done = false;
- initReader();
- }
-
- private void initReader()
- {
- try {
- InputStream resourceStream = this.getClass().getResourceAsStream("/wordcount/word.txt");
- reader = new BufferedReader(new InputStreamReader(resourceStream));
- } catch (Exception ex) {
- throw Throwables.propagate(ex);
- }
- }
-
- @Override
- public void teardown()
- {
- IOUtils.closeQuietly(reader);
- }
-
- @Override
- public void emitTuples()
- {
- if (!done) {
- try {
- String line = reader.readLine();
- if (line == null) {
- done = true;
- reader.close();
- } else {
- this.output.emit(line);
- }
- Thread.sleep(50);
- } catch (IOException ex) {
- throw new RuntimeException(ex);
- } catch (InterruptedException e) {
- throw Throwables.propagate(e);
- }
- }
- }
- }
-
- public static class Collector extends BaseOperator
- {
- private static Map<KeyValPair<Long, String>, Long> result = new HashMap<>();
- private static boolean done = false;
-
- @Override
- public void setup(Context.OperatorContext context)
- {
- super.setup(context);
- done = false;
- }
-
- public static boolean isDone()
- {
- return done;
- }
-
- public static Map<KeyValPair<Long, String>, Long> getResult()
- {
- return result;
- }
-
- public final transient DefaultInputPort<PojoEvent> input = new DefaultInputPort<PojoEvent>()
- {
- @Override
- public void process(PojoEvent tuple)
- {
- result.put(new KeyValPair<Long, String>(tuple.getTimestamp(), tuple.getWord()), tuple.getCount());
- if (tuple.getWord().equals("bye")) {
- done = true;
- }
- }
- };
- }
-
- /**
- * A Pojo Tuple class used for outputting result to JDBC.
- */
- public static class PojoEvent
- {
- private String word;
- private long count;
- private long timestamp;
-
- @Override
- public String toString()
- {
- return "PojoEvent (word=" + getWord() + ", count=" + getCount() + ", timestamp=" + getTimestamp() + ")";
- }
-
- public String getWord()
- {
- return word;
- }
-
- public void setWord(String word)
- {
- this.word = word;
- }
-
- public long getCount()
- {
- return count;
- }
-
- public void setCount(long count)
- {
- this.count = count;
- }
-
- public long getTimestamp()
- {
- return timestamp;
- }
-
- public void setTimestamp(long timestamp)
- {
- this.timestamp = timestamp;
- }
- }
-
- /**
- * A map function that wrap the input string with a random generated timestamp.
- */
- public static class AddTimestampFn implements Function.MapFunction<String, Tuple.TimestampedTuple<String>>
- {
- private static final Duration RAND_RANGE = Duration.standardMinutes(10);
- private final Long minTimestamp;
-
- AddTimestampFn()
- {
- this.minTimestamp = System.currentTimeMillis();
- }
-
- @Override
- public Tuple.TimestampedTuple<String> f(String input)
- {
- // Generate a timestamp that falls somewhere in the past two hours.
- long randMillis = (long)(Math.random() * RAND_RANGE.getMillis());
- long randomTimestamp = minTimestamp + randMillis;
-
- return new Tuple.TimestampedTuple<>(randomTimestamp, input);
- }
- }
-
- /** A MapFunction that converts a Word and Count into a PojoEvent. */
- public static class FormatAsTableRowFn implements Function.MapFunction<Tuple.WindowedTuple<KeyValPair<String, Long>>, PojoEvent>
- {
- @Override
- public PojoEvent f(Tuple.WindowedTuple<KeyValPair<String, Long>> input)
- {
- PojoEvent row = new PojoEvent();
- row.setTimestamp(input.getTimestamp());
- row.setCount(input.getValue().getValue());
- row.setWord(input.getValue().getKey());
- return row;
- }
- }
-
- /**
- * Populate dag with High-Level API.
- * @param dag
- * @param conf
- */
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- TextInput input = new TextInput();
- Collector collector = new Collector();
-
- // Create stream from the TextInput operator.
- ApexStream<Tuple.TimestampedTuple<String>> stream = StreamFactory.fromInput(input, input.output, name("input"))
-
- // Extract all the words from the input line of text.
- .flatMap(new Function.FlatMapFunction<String, String>()
- {
- @Override
- public Iterable<String> f(String input)
- {
- return Arrays.asList(input.split("[\\p{Punct}\\s]+"));
- }
- }, name("ExtractWords"))
-
- // Wrap the word with a randomly generated timestamp.
- .map(new AddTimestampFn(), name("AddTimestampFn"));
-
-
- // apply window and trigger option.
- // TODO: change trigger option to atWaterMark when available.
- WindowedStream<Tuple.TimestampedTuple<String>> windowedWords = stream
- .window(new WindowOption.TimeWindows(Duration.standardMinutes(WINDOW_SIZE)),
- new TriggerOption().accumulatingFiredPanes().withEarlyFiringsAtEvery(1));
-
-
- WindowedStream<PojoEvent> wordCounts =
- // Perform a countByKey transformation to count the appearance of each word in every time window.
- windowedWords.countByKey(new Function.ToKeyValue<Tuple.TimestampedTuple<String>, String, Long>()
- {
- @Override
- public Tuple<KeyValPair<String, Long>> f(Tuple.TimestampedTuple<String> input)
- {
- return new Tuple.TimestampedTuple<KeyValPair<String, Long>>(input.getTimestamp(),
- new KeyValPair<String, Long>(input.getValue(), 1L));
- }
- }, name("count words"))
-
- // Format the output and print out the result.
- .map(new FormatAsTableRowFn(), name("FormatAsTableRowFn")).print(name("console"));
-
- wordCounts.endWith(collector, collector.input, name("Collector")).populateDag(dag);
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/AutoComplete.java
----------------------------------------------------------------------
diff --git a/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/AutoComplete.java b/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/AutoComplete.java
deleted file mode 100644
index 2db59b6..0000000
--- a/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/AutoComplete.java
+++ /dev/null
@@ -1,324 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.apex.malhar.stream.sample.complete;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.apex.malhar.lib.window.TriggerOption;
-import org.apache.apex.malhar.lib.window.Tuple;
-import org.apache.apex.malhar.lib.window.Window;
-import org.apache.apex.malhar.lib.window.WindowOption;
-import org.apache.apex.malhar.stream.api.ApexStream;
-import org.apache.apex.malhar.stream.api.CompositeStreamTransform;
-import org.apache.apex.malhar.stream.api.WindowedStream;
-import org.apache.apex.malhar.stream.api.function.Function;
-import org.apache.apex.malhar.stream.api.impl.StreamFactory;
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.conf.Configuration;
-
-import com.google.common.base.Throwables;
-
-import com.datatorrent.api.Context.OperatorContext;
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.DefaultInputPort;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.api.InputOperator;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-import com.datatorrent.common.util.BaseOperator;
-import com.datatorrent.lib.util.KeyValPair;
-
-import static org.apache.apex.malhar.stream.api.Option.Options.name;
-
-/**
- * An example that computes the most popular hash tags
- * for every prefix, which can be used for auto-completion.
- * This application is identical to TwitterAutoComplete, except it's
- * reading from a file. This application is mainly for local testing
- * purpose.
- *
- * <p>This will update the datastore every 10 seconds based on the last
- * 30 minutes of data received.
- *
- * @since 3.5.0
- */
-@ApplicationAnnotation(name = "AutoComplete")
-public class AutoComplete implements StreamingApplication
-{
-
- /**
- * A dummy Twitter input operator. It reads from a text file containing some tweets and output a line every
- * half of a second.
- */
- public static class TweetsInput extends BaseOperator implements InputOperator
- {
- public final transient DefaultOutputPort<String> output = new DefaultOutputPort<>();
- private boolean done;
-
- private transient BufferedReader reader;
-
- @Override
- public void setup(OperatorContext context)
- {
- done = false;
- initReader();
- }
-
- private void initReader()
- {
- try {
- InputStream resourceStream = this.getClass().getResourceAsStream("/sampletweets.txt");
- reader = new BufferedReader(new InputStreamReader(resourceStream));
- } catch (Exception ex) {
- throw Throwables.propagate(ex);
- }
- }
-
- @Override
- public void teardown()
- {
- IOUtils.closeQuietly(reader);
- }
-
- @Override
- public void emitTuples()
- {
- if (!done) {
- try {
- String line = reader.readLine();
- if (line == null) {
- done = true;
- reader.close();
- } else {
- this.output.emit(line);
- }
- Thread.sleep(50);
- } catch (IOException ex) {
- throw new RuntimeException(ex);
- } catch (InterruptedException e) {
- // Ignore it.
- }
- }
- }
- }
-
- public static class Collector extends BaseOperator
- {
- private static Map<String, List<CompletionCandidate>> result = new HashMap<>();
- private static boolean done = false;
-
- public static boolean isDone()
- {
- return done;
- }
-
- @Override
- public void setup(OperatorContext context)
- {
- super.setup(context);
- done = false;
- }
-
- public static Map<String, List<CompletionCandidate>> getResult()
- {
- return result;
- }
-
- public final transient DefaultInputPort<Tuple.WindowedTuple<KeyValPair<String, List<CompletionCandidate>>>> input = new DefaultInputPort<Tuple.WindowedTuple<KeyValPair<String, List<CompletionCandidate>>>>()
- {
- @Override
- public void process(Tuple.WindowedTuple<KeyValPair<String, List<CompletionCandidate>>> tuple)
- {
- if (tuple.getValue().getKey().equals("yarn")) {
- done = true;
- }
- result.put(tuple.getValue().getKey(), tuple.getValue().getValue());
- }
- };
- }
-
- /**
- * FlapMap Function to extract all hashtags from a text form tweet.
- */
- private static class ExtractHashtags implements Function.FlatMapFunction<String, String>
- {
-
- @Override
- public Iterable<String> f(String input)
- {
- List<String> result = new LinkedList<>();
- Matcher m = Pattern.compile("#\\S+").matcher(input);
- while (m.find()) {
- result.add(m.group().substring(1));
- }
- return result;
- }
- }
-
- /**
- * Lower latency, but more expensive.
- */
- private static class ComputeTopFlat
- extends CompositeStreamTransform<WindowedStream<CompletionCandidate>, WindowedStream<Tuple.WindowedTuple<KeyValPair<String, List<CompletionCandidate>>>>>
- {
- private final int candidatesPerPrefix;
- private final int minPrefix;
-
- public ComputeTopFlat(int candidatesPerPrefix, int minPrefix)
- {
- this.candidatesPerPrefix = candidatesPerPrefix;
- this.minPrefix = minPrefix;
- }
-
- @Override
- public WindowedStream<Tuple.WindowedTuple<KeyValPair<String, List<CompletionCandidate>>>> compose(
- WindowedStream<CompletionCandidate> input)
- {
- return input
- .<KeyValPair<String, CompletionCandidate>, WindowedStream<KeyValPair<String, CompletionCandidate>>>flatMap(new AllPrefixes(minPrefix))
- .accumulateByKey(new TopNByKey(), new Function.ToKeyValue<KeyValPair<String, CompletionCandidate>, String,
- CompletionCandidate>()
- {
- @Override
- public Tuple<KeyValPair<String, CompletionCandidate>> f(KeyValPair<String, CompletionCandidate> tuple)
- {
- // TODO: Should be removed after Auto-wrapping is supported.
- return new Tuple.WindowedTuple<>(Window.GlobalWindow.INSTANCE, tuple);
- }
- });
- }
- }
-
- /**
- * FlapMap Function to extract all prefixes of the hashtag in the input CompletionCandidate, and output
- * KeyValPairs of the prefix and the CompletionCandidate
- */
- private static class AllPrefixes implements Function.FlatMapFunction<CompletionCandidate, KeyValPair<String, CompletionCandidate>>
- {
- private final int minPrefix;
- private final int maxPrefix;
-
- public AllPrefixes()
- {
- this(0, Integer.MAX_VALUE);
- }
-
- public AllPrefixes(int minPrefix)
- {
- this(minPrefix, Integer.MAX_VALUE);
- }
-
- public AllPrefixes(int minPrefix, int maxPrefix)
- {
- this.minPrefix = minPrefix;
- this.maxPrefix = maxPrefix;
- }
-
- @Override
- public Iterable<KeyValPair<String, CompletionCandidate>> f(CompletionCandidate input)
- {
- List<KeyValPair<String, CompletionCandidate>> result = new LinkedList<>();
- String word = input.getValue();
- for (int i = minPrefix; i <= Math.min(word.length(), maxPrefix); i++) {
-
- result.add(new KeyValPair<>(input.getValue().substring(0, i).toLowerCase(), input));
- }
- return result;
- }
- }
-
- /**
- * A Composite stream transform that takes as input a list of tokens and returns
- * the most common tokens per prefix.
- */
- public static class ComputeTopCompletions
- extends CompositeStreamTransform<WindowedStream<String>, WindowedStream<Tuple.WindowedTuple<KeyValPair<String, List<CompletionCandidate>>>>>
- {
- private final int candidatesPerPrefix;
- private final boolean recursive;
-
- protected ComputeTopCompletions(int candidatesPerPrefix, boolean recursive)
- {
- this.candidatesPerPrefix = candidatesPerPrefix;
- this.recursive = recursive;
- }
-
- public static ComputeTopCompletions top(int candidatesPerPrefix, boolean recursive)
- {
- return new ComputeTopCompletions(candidatesPerPrefix, recursive);
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public WindowedStream<Tuple.WindowedTuple<KeyValPair<String, List<CompletionCandidate>>>> compose(WindowedStream<String> inputStream)
- {
- ApexStream<CompletionCandidate> candidates = inputStream
- .countByKey(new Function.ToKeyValue<String, String, Long>()
- {
- @Override
- public Tuple<KeyValPair<String, Long>> f(String input)
- {
- return new Tuple.PlainTuple<>(new KeyValPair<>(input, 1L));
- }
- }, name("countByKey"))
- .map(new Function.MapFunction<Tuple.WindowedTuple<KeyValPair<String,Long>>, CompletionCandidate>()
- {
- @Override
- public CompletionCandidate f(Tuple.WindowedTuple<KeyValPair<String, Long>> input)
- {
- return new CompletionCandidate(input.getValue().getKey(), input.getValue().getValue());
- }
- }, name("ToCompletionCandidate"));
-
- return candidates.addCompositeStreams(new ComputeTopFlat(10, 1));
-
- }
- }
-
- /**
- * Populate the dag with High-Level API.
- * @param dag
- * @param conf
- */
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- TweetsInput input = new TweetsInput();
- Collector collector = new Collector();
-
- WindowOption windowOption = new WindowOption.GlobalWindow();
-
- ApexStream<String> tags = StreamFactory.fromInput(input, input.output, name("tweetSampler"))
- .flatMap(new ExtractHashtags());
-
- tags.window(windowOption, new TriggerOption().accumulatingFiredPanes().withEarlyFiringsAtEvery(1))
- .addCompositeStreams(ComputeTopCompletions.top(10, true)).print(name("console"))
- .endWith(collector, collector.input, name("collector"))
- .populateDag(dag);
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/CompletionCandidate.java
----------------------------------------------------------------------
diff --git a/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/CompletionCandidate.java b/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/CompletionCandidate.java
deleted file mode 100644
index 991424e..0000000
--- a/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/CompletionCandidate.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.apex.malhar.stream.sample.complete;
-
-/**
- * Class used to store tag-count pairs in Auto Complete Demo.
- *
- * @since 3.5.0
- */
-public class CompletionCandidate implements Comparable<CompletionCandidate>
-{
- private long count;
- private String value;
-
- public CompletionCandidate(String value, long count)
- {
- this.value = value;
- this.count = count;
- }
-
- public long getCount()
- {
- return count;
- }
-
- public String getValue()
- {
- return value;
- }
-
- // Empty constructor required for Kryo.
- public CompletionCandidate()
- {
-
- }
-
- @Override
- public int compareTo(CompletionCandidate o)
- {
- if (this.count < o.count) {
- return -1;
- } else if (this.count == o.count) {
- return this.value.compareTo(o.value);
- } else {
- return 1;
- }
- }
-
- @Override
- public boolean equals(Object other)
- {
- if (other instanceof CompletionCandidate) {
- CompletionCandidate that = (CompletionCandidate)other;
- return this.count == that.count && this.value.equals(that.value);
- } else {
- return false;
- }
- }
-
- @Override
- public int hashCode()
- {
- return Long.valueOf(count).hashCode() ^ value.hashCode();
- }
-
- @Override
- public String toString()
- {
- return "CompletionCandidate[" + value + ", " + count + "]";
- }
-}
-
-
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/PojoEvent.java
----------------------------------------------------------------------
diff --git a/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/PojoEvent.java b/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/PojoEvent.java
deleted file mode 100644
index ee15d90..0000000
--- a/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/PojoEvent.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.apex.malhar.stream.sample.complete;
-
-/**
- * Tuple Class for JdbcOutput of StreamingWordExtract.
- *
- * @since 3.5.0
- */
-public class PojoEvent extends Object
-{
- private String stringValue;
-
- @Override
- public String toString()
- {
- return "PojoEvent [stringValue=" + getStringValue() + "]";
- }
-
- public void setStringValue(String newString)
- {
- this.stringValue = newString;
- }
-
- public String getStringValue()
- {
- return this.stringValue;
- }
-}
-
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/StreamingWordExtract.java
----------------------------------------------------------------------
diff --git a/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/StreamingWordExtract.java b/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/StreamingWordExtract.java
deleted file mode 100644
index 07f01d0..0000000
--- a/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/StreamingWordExtract.java
+++ /dev/null
@@ -1,162 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.apex.malhar.stream.sample.complete;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.apex.malhar.stream.api.ApexStream;
-import org.apache.apex.malhar.stream.api.Option;
-import org.apache.apex.malhar.stream.api.function.Function;
-import org.apache.apex.malhar.stream.api.impl.StreamFactory;
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-import com.datatorrent.lib.db.jdbc.JdbcFieldInfo;
-import com.datatorrent.lib.db.jdbc.JdbcPOJOInsertOutputOperator;
-import com.datatorrent.lib.db.jdbc.JdbcTransactionalStore;
-
-import static java.sql.Types.VARCHAR;
-
-/**
- * Beam StreamingWordExtract Example.
- *
- * @since 3.5.0
- */
-@ApplicationAnnotation(name = "StreamingWordExtract")
-public class StreamingWordExtract implements StreamingApplication
-{
- private static int wordCount = 0; // A counter to count number of words have been extracted.
- private static int entriesMapped = 0; // A counter to count number of entries have been mapped.
-
- public int getWordCount()
- {
- return wordCount;
- }
-
- public int getEntriesMapped()
- {
- return entriesMapped;
- }
-
- /**
- * A MapFunction that tokenizes lines of text into individual words.
- */
- public static class ExtractWords implements Function.FlatMapFunction<String, String>
- {
- @Override
- public Iterable<String> f(String input)
- {
- List<String> result = new ArrayList<>(Arrays.asList(input.split("[^a-zA-Z0-9']+")));
- wordCount += result.size();
- return result;
- }
- }
-
-
- /**
- * A MapFunction that uppercases a word.
- */
- public static class Uppercase implements Function.MapFunction<String, String>
- {
- @Override
- public String f(String input)
- {
- return input.toUpperCase();
- }
- }
-
-
- /**
- * A filter function to filter out empty strings.
- */
- public static class EmptyStringFilter implements Function.FilterFunction<String>
- {
- @Override
- public boolean f(String input)
- {
- return !input.isEmpty();
- }
- }
-
-
- /**
- * A map function to map the result string to a pojo entry.
- */
- public static class PojoMapper implements Function.MapFunction<String, Object>
- {
-
- @Override
- public Object f(String input)
- {
- PojoEvent pojo = new PojoEvent();
- pojo.setStringValue(input);
- entriesMapped++;
- return pojo;
- }
- }
-
- /**
- * Add field infos to the {@link JdbcPOJOInsertOutputOperator}.
- */
- private static List<JdbcFieldInfo> addFieldInfos()
- {
- List<JdbcFieldInfo> fieldInfos = new ArrayList<>();
- fieldInfos.add(new JdbcFieldInfo("STRINGVALUE", "stringValue", JdbcFieldInfo.SupportType.STRING, VARCHAR));
- return fieldInfos;
- }
-
- /**
- * Populate dag with High-Level API.
- * @param dag
- * @param conf
- */
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- JdbcPOJOInsertOutputOperator jdbcOutput = new JdbcPOJOInsertOutputOperator();
- jdbcOutput.setFieldInfos(addFieldInfos());
- JdbcTransactionalStore outputStore = new JdbcTransactionalStore();
- jdbcOutput.setStore(outputStore);
- jdbcOutput.setTablename("TestTable");
-
- // Create a stream reading from a folder.
- ApexStream<String> stream = StreamFactory.fromFolder("./src/test/resources/data");
-
- // Extract all the words from the input line of text.
- stream.flatMap(new ExtractWords())
-
- // Filter out the empty strings.
- .filter(new EmptyStringFilter())
-
- // Change every word to uppercase.
- .map(new Uppercase())
-
- // Map the resulted word to a Pojo entry.
- .map(new PojoMapper())
-
- // Output the entries to JdbcOutput and insert them into a table.
- .endWith(jdbcOutput, jdbcOutput.input, Option.Options.name("jdbcOutput"));
-
- stream.populateDag(dag);
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/TopNByKey.java
----------------------------------------------------------------------
diff --git a/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/TopNByKey.java b/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/TopNByKey.java
deleted file mode 100644
index c7ccae3..0000000
--- a/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/TopNByKey.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.apex.malhar.stream.sample.complete;
-
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.apex.malhar.lib.window.Accumulation;
-
-/**
- * Specialized TopNByKey accumulation for AutoComplete Demo.
- *
- * @since 3.5.0
- */
-public class TopNByKey implements
- Accumulation<CompletionCandidate, Map<String, Long>, List<CompletionCandidate>>
-{
- int n = 10;
-
- Comparator comparator;
-
- public void setN(int n)
- {
- this.n = n;
- }
-
- public void setComparator(Comparator comparator)
- {
- this.comparator = comparator;
- }
-
- @Override
- public Map<String, Long> defaultAccumulatedValue()
- {
- return new HashMap<>();
- }
-
- /**
- * Accumulate the input. Update the entry in the Accumulation Map if the key of the input is existed, create a
- * new entry otherwise.
- * @param accumulatedValue
- * @param input
- * @return
- */
- @Override
- public Map<String, Long> accumulate(Map<String, Long> accumulatedValue, CompletionCandidate input)
- {
- accumulatedValue.put(input.getValue(), input.getCount());
- return accumulatedValue;
- }
-
- /**
- * Merge two Maps together. For every key, keep the larger value in the resulted Map.
- * @param accumulatedValue1
- * @param accumulatedValue2
- * @return
- */
- @Override
- public Map<String, Long> merge(Map<String, Long> accumulatedValue1, Map<String, Long> accumulatedValue2)
- {
- for (Map.Entry<String, Long> entry : accumulatedValue2.entrySet()) {
- if (accumulatedValue1.containsKey(entry.getKey()) && accumulatedValue1.get(entry.getKey()) > entry.getValue()) {
- continue;
- }
- accumulatedValue1.put(entry.getKey(), entry.getValue());
- }
- return accumulatedValue1;
- }
-
- /**
- * Loop through the Accumulation Map to get the top n entries based on their values, return a list containing
- * those entries.
- * @param accumulatedValue
- * @return
- */
- @Override
- public List<CompletionCandidate> getOutput(Map<String, Long> accumulatedValue)
- {
- LinkedList<CompletionCandidate> result = new LinkedList<>();
- for (Map.Entry<String, Long> entry : accumulatedValue.entrySet()) {
- int k = 0;
- for (CompletionCandidate inMemory : result) {
- if (entry.getValue() > inMemory.getCount()) {
- break;
- }
- k++;
- }
- result.add(k, new CompletionCandidate(entry.getKey(), entry.getValue()));
- if (result.size() > n) {
- result.remove(result.get(result.size() - 1));
- }
- }
- return result;
- }
-
- @Override
- public List<CompletionCandidate> getRetraction(List<CompletionCandidate> value)
- {
- return new LinkedList<>();
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/TopWikipediaSessions.java
----------------------------------------------------------------------
diff --git a/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/TopWikipediaSessions.java b/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/TopWikipediaSessions.java
deleted file mode 100644
index 68ec733..0000000
--- a/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/TopWikipediaSessions.java
+++ /dev/null
@@ -1,347 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.apex.malhar.stream.sample.complete;
-
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.List;
-import java.util.Random;
-import javax.annotation.Nullable;
-
-import org.joda.time.Duration;
-
-import org.apache.apex.malhar.lib.window.TriggerOption;
-import org.apache.apex.malhar.lib.window.Tuple;
-import org.apache.apex.malhar.lib.window.Window;
-import org.apache.apex.malhar.lib.window.WindowOption;
-import org.apache.apex.malhar.lib.window.accumulation.TopN;
-import org.apache.apex.malhar.stream.api.ApexStream;
-import org.apache.apex.malhar.stream.api.CompositeStreamTransform;
-import org.apache.apex.malhar.stream.api.WindowedStream;
-import org.apache.apex.malhar.stream.api.function.Function;
-import org.apache.apex.malhar.stream.api.impl.StreamFactory;
-import org.apache.hadoop.conf.Configuration;
-
-
-import com.datatorrent.api.Context;
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.DefaultInputPort;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.api.InputOperator;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-import com.datatorrent.common.util.BaseOperator;
-import com.datatorrent.lib.util.KeyValPair;
-
-import static org.apache.apex.malhar.stream.api.Option.Options.name;
-
-/**
- * Beam's TopWikipediaSessions Example.
- *
- * @since 3.5.0
- */
-@ApplicationAnnotation(name = "TopWikipediaSessions")
-public class TopWikipediaSessions implements StreamingApplication
-{
- /**
- * A generator that outputs a stream of combinations of some users and some randomly generated edit time.
- */
- public static class SessionGen extends BaseOperator implements InputOperator
- {
- private String[] names = new String[]{"user1", "user2", "user3", "user4"};
- public transient DefaultOutputPort<KeyValPair<String, Long>> output = new DefaultOutputPort<>();
-
- private static final Duration RAND_RANGE = Duration.standardDays(365);
- private Long minTimestamp;
- private long sleepTime;
- private static int tupleCount = 0;
-
- public static int getTupleCount()
- {
- return tupleCount;
- }
-
- private String randomName(String[] names)
- {
- int index = new Random().nextInt(names.length);
- return names[index];
- }
-
- @Override
- public void setup(Context.OperatorContext context)
- {
- super.setup(context);
- tupleCount = 0;
- minTimestamp = System.currentTimeMillis();
- sleepTime = context.getValue(Context.OperatorContext.SPIN_MILLIS);
- }
-
- @Override
- public void emitTuples()
- {
- long randMillis = (long)(Math.random() * RAND_RANGE.getMillis());
- long randomTimestamp = minTimestamp + randMillis;
- output.emit(new KeyValPair<String, Long>(randomName(names), randomTimestamp));
- tupleCount++;
- try {
- Thread.sleep(sleepTime);
- } catch (InterruptedException e) {
- // Ignore it.
- }
- }
- }
-
- public static class Collector extends BaseOperator
- {
- private final int resultSize = 5;
- private static List<List<TempWrapper>> result = new ArrayList<>();
-
- public static List<List<TempWrapper>> getResult()
- {
- return result;
- }
-
- public final transient DefaultInputPort<Tuple.WindowedTuple<List<TempWrapper>>> input = new DefaultInputPort<Tuple.WindowedTuple<List<TempWrapper>>>()
- {
- @Override
- public void process(Tuple.WindowedTuple<List<TempWrapper>> tuple)
- {
- if (result.size() == resultSize) {
- result.remove(0);
- }
- result.add(tuple.getValue());
- }
- };
- }
-
-
- /**
- * Convert the upstream (user, time) combination to a timestamped tuple of user.
- */
- static class ExtractUserAndTimestamp implements Function.MapFunction<KeyValPair<String, Long>, Tuple.TimestampedTuple<String>>
- {
- @Override
- public Tuple.TimestampedTuple<String> f(KeyValPair<String, Long> input)
- {
- long timestamp = input.getValue();
- String userName = input.getKey();
-
- // Sets the implicit timestamp field to be used in windowing.
- return new Tuple.TimestampedTuple<>(timestamp, userName);
-
- }
- }
-
- /**
- * Computes the number of edits in each user session. A session is defined as
- * a string of edits where each is separated from the next by less than an hour.
- */
- static class ComputeSessions
- extends CompositeStreamTransform<ApexStream<Tuple.TimestampedTuple<String>>, WindowedStream<Tuple.WindowedTuple<KeyValPair<String, Long>>>>
- {
- @Override
- public WindowedStream<Tuple.WindowedTuple<KeyValPair<String, Long>>> compose(ApexStream<Tuple.TimestampedTuple<String>> inputStream)
- {
- return inputStream
-
- // Chuck the stream into session windows.
- .window(new WindowOption.SessionWindows(Duration.standardHours(1)), new TriggerOption().accumulatingFiredPanes().withEarlyFiringsAtEvery(1))
-
- // Count the number of edits for a user within one session.
- .countByKey(new Function.ToKeyValue<Tuple.TimestampedTuple<String>, String, Long>()
- {
- @Override
- public Tuple.TimestampedTuple<KeyValPair<String, Long>> f(Tuple.TimestampedTuple<String> input)
- {
- return new Tuple.TimestampedTuple<KeyValPair<String, Long>>(input.getTimestamp(), new KeyValPair<String, Long>(input.getValue(), 1L));
- }
- }, name("ComputeSessions"));
- }
- }
-
- /**
- * A comparator class used for comparing two TempWrapper objects.
- */
- public static class Comp implements Comparator<TempWrapper>
- {
- @Override
- public int compare(TempWrapper o1, TempWrapper o2)
- {
- return Long.compare(o1.getValue().getValue(), o2.getValue().getValue());
- }
- }
-
- /**
- * A function to extract timestamp from a TempWrapper object.
- */
- // TODO: Need to revisit and change back to using TimestampedTuple.
- public static class TimestampExtractor implements com.google.common.base.Function<TempWrapper, Long>
- {
- @Override
- public Long apply(@Nullable TempWrapper input)
- {
- return input.getTimestamp();
- }
- }
-
- /**
- * A temporary wrapper to wrap a KeyValPair and a timestamp together to represent a timestamped tuple, the reason
- * for this is that we cannot resolve a type conflict when calling accumulate(). After the issue resolved, we can
- * remove this class.
- */
- public static class TempWrapper
- {
- private KeyValPair<String, Long> value;
- private Long timestamp;
-
- public TempWrapper()
- {
-
- }
-
- public TempWrapper(KeyValPair<String, Long> value, Long timestamp)
- {
- this.value = value;
- this.timestamp = timestamp;
- }
-
- @Override
- public String toString()
- {
- return this.value + " - " + this.timestamp;
- }
-
- public Long getTimestamp()
- {
- return timestamp;
- }
-
- public void setTimestamp(Long timestamp)
- {
- this.timestamp = timestamp;
- }
-
- public KeyValPair<String, Long> getValue()
- {
- return value;
- }
-
- public void setValue(KeyValPair<String, Long> value)
- {
- this.value = value;
- }
- }
-
- /**
- * Computes the longest session ending in each month, in this case we use 30 days to represent every month.
- */
- private static class TopPerMonth
- extends CompositeStreamTransform<ApexStream<Tuple.WindowedTuple<KeyValPair<String, Long>>>, WindowedStream<Tuple.WindowedTuple<List<TempWrapper>>>>
- {
-
- @Override
- public WindowedStream<Tuple.WindowedTuple<List<TempWrapper>>> compose(ApexStream<Tuple.WindowedTuple<KeyValPair<String, Long>>> inputStream)
- {
- TopN<TempWrapper> topN = new TopN<>();
- topN.setN(10);
- topN.setComparator(new Comp());
-
- return inputStream
-
- // Map the input WindowedTuple to a TempWrapper object.
- .map(new Function.MapFunction<Tuple.WindowedTuple<KeyValPair<String, Long>>, TempWrapper>()
- {
- @Override
- public TempWrapper f(Tuple.WindowedTuple<KeyValPair<String, Long>> input)
- {
- Window window = input.getWindows().iterator().next();
- return new TempWrapper(input.getValue(), window.getBeginTimestamp());
- }
- }, name("TempWrapper"))
-
- // Apply window and trigger option again, this time chuck the stream into fixed time windows.
- .window(new WindowOption.TimeWindows(Duration.standardDays(30)), new TriggerOption().accumulatingFiredPanes().withEarlyFiringsAtEvery(Duration.standardSeconds(5)))
-
- // Compute the top 10 user-sessions with most number of edits.
- .accumulate(topN, name("TopN")).with("timestampExtractor", new TimestampExtractor());
- }
- }
-
- /**
- * A map function that combine the user and his/her edit session together to a string and use that string as a key
- * with number of edits in that session as value to create a new key value pair to send to downstream.
- */
- static class SessionsToStringsDoFn implements Function.MapFunction<Tuple.WindowedTuple<KeyValPair<String, Long>>, Tuple.WindowedTuple<KeyValPair<String, Long>>>
- {
- @Override
- public Tuple.WindowedTuple<KeyValPair<String, Long>> f(Tuple.WindowedTuple<KeyValPair<String, Long>> input)
- {
- Window window = input.getWindows().iterator().next();
- return new Tuple.WindowedTuple<KeyValPair<String, Long>>(window, new KeyValPair<String, Long>(
- input.getValue().getKey() + " : " + window.getBeginTimestamp() + " : " + window.getDurationMillis(),
- input.getValue().getValue()));
- }
- }
-
- /**
- * A flatmap function that turns the result into readable format.
- */
- static class FormatOutputDoFn implements Function.FlatMapFunction<Tuple.WindowedTuple<List<TempWrapper>>, String>
- {
- @Override
- public Iterable<String> f(Tuple.WindowedTuple<List<TempWrapper>> input)
- {
- ArrayList<String> result = new ArrayList<>();
- for (TempWrapper item : input.getValue()) {
- String session = item.getValue().getKey();
- long count = item.getValue().getValue();
- Window window = input.getWindows().iterator().next();
- result.add(session + " + " + count + " : " + window.getBeginTimestamp());
- }
- return result;
- }
- }
-
- /**
- * A composite transform that compute the top wikipedia sessions.
- */
- public static class ComputeTopSessions extends CompositeStreamTransform<ApexStream<KeyValPair<String, Long>>, WindowedStream<Tuple.WindowedTuple<List<TempWrapper>>>>
- {
- @Override
- public WindowedStream<Tuple.WindowedTuple<List<TempWrapper>>> compose(ApexStream<KeyValPair<String, Long>> inputStream)
- {
- return inputStream
- .map(new ExtractUserAndTimestamp(), name("ExtractUserAndTimestamp"))
- .addCompositeStreams(new ComputeSessions())
- .map(new SessionsToStringsDoFn(), name("SessionsToStringsDoFn"))
- .addCompositeStreams(new TopPerMonth());
- }
- }
-
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- SessionGen sg = new SessionGen();
- Collector collector = new Collector();
- StreamFactory.fromInput(sg, sg.output, name("sessionGen"))
- .addCompositeStreams(new ComputeTopSessions())
- .print(name("console"))
- .endWith(collector, collector.input, name("collector")).populateDag(dag);
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/TrafficRoutes.java
----------------------------------------------------------------------
diff --git a/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/TrafficRoutes.java b/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/TrafficRoutes.java
deleted file mode 100644
index e6a53d6..0000000
--- a/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/TrafficRoutes.java
+++ /dev/null
@@ -1,523 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.apex.malhar.stream.sample.complete;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Hashtable;
-import java.util.List;
-import java.util.Map;
-
-import javax.annotation.Nullable;
-
-import org.joda.time.Duration;
-
-import org.apache.apex.malhar.lib.window.TriggerOption;
-import org.apache.apex.malhar.lib.window.Tuple;
-import org.apache.apex.malhar.lib.window.WindowOption;
-import org.apache.apex.malhar.lib.window.accumulation.Group;
-import org.apache.apex.malhar.stream.api.ApexStream;
-import org.apache.apex.malhar.stream.api.CompositeStreamTransform;
-import org.apache.apex.malhar.stream.api.WindowedStream;
-import org.apache.apex.malhar.stream.api.function.Function;
-import org.apache.apex.malhar.stream.api.impl.StreamFactory;
-import org.apache.hadoop.conf.Configuration;
-
-import com.google.common.collect.Lists;
-
-import com.datatorrent.api.Context;
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.DefaultInputPort;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.api.InputOperator;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-import com.datatorrent.common.util.BaseOperator;
-import com.datatorrent.lib.util.KeyValPair;
-
-import static org.apache.apex.malhar.stream.api.Option.Options.name;
-
-/**
- * Beam's TrafficRoutes example.
- *
- * @since 3.5.0
- */
-@ApplicationAnnotation(name = "TrafficRoutes")
-public class TrafficRoutes implements StreamingApplication
-{
- static Map<String, String> sdStations = buildStationInfo();
- static final int WINDOW_DURATION = 3; // Default sliding window duration in minutes
- static final int WINDOW_SLIDE_EVERY = 1; // Default window 'slide every' setting in minutes
-
- /**
- * This class holds information about a station reading's average speed.
- */
- public static class StationSpeed implements Comparable<StationSpeed>
- {
- @Nullable
- String stationId;
- @Nullable
- Double avgSpeed;
- @Nullable
- Long timestamp;
-
- public StationSpeed() {}
-
- public StationSpeed(String stationId, Double avgSpeed, Long timestamp)
- {
- this.stationId = stationId;
- this.avgSpeed = avgSpeed;
- this.timestamp = timestamp;
- }
-
- public void setAvgSpeed(@Nullable Double avgSpeed)
- {
- this.avgSpeed = avgSpeed;
- }
-
- public void setStationId(@Nullable String stationId)
- {
- this.stationId = stationId;
- }
-
- public void setTimestamp(@Nullable Long timestamp)
- {
- this.timestamp = timestamp;
- }
-
- @Nullable
- public Long getTimestamp()
- {
- return timestamp;
- }
-
- public String getStationId()
- {
- return this.stationId;
- }
-
- public Double getAvgSpeed()
- {
- return this.avgSpeed;
- }
-
- @Override
- public int compareTo(StationSpeed other)
- {
- return Long.compare(this.timestamp, other.timestamp);
- }
- }
-
- /**
- * This class holds information about a route's speed/slowdown.
- */
- static class RouteInfo
- {
- @Nullable
- String route;
- @Nullable
- Double avgSpeed;
- @Nullable
- Boolean slowdownEvent;
-
- public RouteInfo()
- {
-
- }
-
- public RouteInfo(String route, Double avgSpeed, Boolean slowdownEvent)
- {
- this.route = route;
- this.avgSpeed = avgSpeed;
- this.slowdownEvent = slowdownEvent;
- }
-
- public String getRoute()
- {
- return this.route;
- }
-
- public Double getAvgSpeed()
- {
- return this.avgSpeed;
- }
-
- public Boolean getSlowdownEvent()
- {
- return this.slowdownEvent;
- }
- }
-
- /**
- * Extract the timestamp field from the input string, and wrap the input string in a {@link Tuple.TimestampedTuple}
- * with the extracted timestamp.
- */
- static class ExtractTimestamps implements Function.MapFunction<String, Tuple.TimestampedTuple<String>>
- {
-
- @Override
- public Tuple.TimestampedTuple<String> f(String input)
- {
- String[] items = input.split(",");
- String timestamp = tryParseTimestamp(items);
-
- return new Tuple.TimestampedTuple<>(Long.parseLong(timestamp), input);
- }
- }
-
- /**
- * Filter out readings for the stations along predefined 'routes', and output
- * (station, speed info) keyed on route.
- */
- static class ExtractStationSpeedFn implements Function.FlatMapFunction<Tuple.TimestampedTuple<String>, KeyValPair<String, StationSpeed>>
- {
-
- @Override
- public Iterable<KeyValPair<String, StationSpeed>> f(Tuple.TimestampedTuple<String> input)
- {
-
- ArrayList<KeyValPair<String, StationSpeed>> result = new ArrayList<>();
- String[] items = input.getValue().split(",");
- String stationType = tryParseStationType(items);
- // For this analysis, use only 'main line' station types
- if (stationType != null && stationType.equals("ML")) {
- Double avgSpeed = tryParseAvgSpeed(items);
- String stationId = tryParseStationId(items);
- // For this simple example, filter out everything but some hardwired routes.
- if (avgSpeed != null && stationId != null && sdStations.containsKey(stationId)) {
- StationSpeed stationSpeed =
- new StationSpeed(stationId, avgSpeed, input.getTimestamp());
- // The tuple key is the 'route' name stored in the 'sdStations' hash.
- KeyValPair<String, StationSpeed> outputValue = new KeyValPair<>(sdStations.get(stationId), stationSpeed);
- result.add(outputValue);
- }
- }
- return result;
- }
- }
-
- /**
- * For a given route, track average speed for the window. Calculate whether
- * traffic is currently slowing down, via a predefined threshold. If a supermajority of
- * speeds in this sliding window are less than the previous reading we call this a 'slowdown'.
- * Note: these calculations are for example purposes only, and are unrealistic and oversimplified.
- */
- static class GatherStats
- implements Function.FlatMapFunction<Tuple.WindowedTuple<KeyValPair<String, List<StationSpeed>>>, Tuple.TimestampedTuple<KeyValPair<String, RouteInfo>>>
- {
- @Override
- public Iterable<Tuple.TimestampedTuple<KeyValPair<String, RouteInfo>>> f(Tuple.WindowedTuple<KeyValPair<String, List<StationSpeed>>> input)
- {
- ArrayList<Tuple.TimestampedTuple<KeyValPair<String, RouteInfo>>> result = new ArrayList<>();
- String route = input.getValue().getKey();
- double speedSum = 0.0;
- int speedCount = 0;
- int speedups = 0;
- int slowdowns = 0;
- List<StationSpeed> infoList = Lists.newArrayList(input.getValue().getValue());
- // StationSpeeds sort by embedded timestamp.
- Collections.sort(infoList);
- Map<String, Double> prevSpeeds = new HashMap<>();
- // For all stations in the route, sum (non-null) speeds. Keep a count of the non-null speeds.
- for (StationSpeed item : infoList) {
- Double speed = item.getAvgSpeed();
- if (speed != null) {
- speedSum += speed;
- speedCount++;
- Double lastSpeed = prevSpeeds.get(item.getStationId());
- if (lastSpeed != null) {
- if (lastSpeed < speed) {
- speedups += 1;
- } else {
- slowdowns += 1;
- }
- }
- prevSpeeds.put(item.getStationId(), speed);
- }
- }
- if (speedCount == 0) {
- // No average to compute.
- return result;
- }
- double speedAvg = speedSum / speedCount;
- boolean slowdownEvent = slowdowns >= 2 * speedups;
- RouteInfo routeInfo = new RouteInfo(route, speedAvg, slowdownEvent);
- result.add(new Tuple.TimestampedTuple<KeyValPair<String, RouteInfo>>(input.getTimestamp(), new KeyValPair<String, RouteInfo>(route, routeInfo)));
- return result;
- }
- }
-
- /**
- * Output Pojo class for outputting result to JDBC.
- */
- static class OutputPojo
- {
- private Double avgSpeed;
- private Boolean slowdownEvent;
- private String key;
- private Long timestamp;
-
- public OutputPojo()
- {
- }
-
- public OutputPojo(Double avgSpeed, Boolean slowdownEvent, String key, Long timestamp)
- {
- this.avgSpeed = avgSpeed;
- this.slowdownEvent = slowdownEvent;
- this.key = key;
- this.timestamp = timestamp;
- }
-
- @Override
- public String toString()
- {
- return key + " + " + avgSpeed + " + " + slowdownEvent + " + " + timestamp;
- }
-
- public void setTimestamp(Long timestamp)
- {
- this.timestamp = timestamp;
- }
-
- public Long getTimestamp()
- {
- return timestamp;
- }
-
- public void setAvgSpeed(Double avgSpeed)
- {
- this.avgSpeed = avgSpeed;
- }
-
- public Double getAvgSpeed()
- {
- return avgSpeed;
- }
-
- public void setKey(String key)
- {
- this.key = key;
- }
-
- public String getKey()
- {
- return key;
- }
-
- public void setSlowdownEvent(Boolean slowdownEvent)
- {
- this.slowdownEvent = slowdownEvent;
- }
-
- public Boolean getSlowdownEvent()
- {
- return slowdownEvent;
- }
-
- }
-
- public static class Collector extends BaseOperator
- {
- private static Map<KeyValPair<Long, String>, KeyValPair<Double, Boolean>> result = new HashMap<>();
-
- public static Map<KeyValPair<Long, String>, KeyValPair<Double, Boolean>> getResult()
- {
- return result;
- }
-
- public final transient DefaultInputPort<OutputPojo> input = new DefaultInputPort<OutputPojo>()
- {
- @Override
- public void process(OutputPojo tuple)
- {
- result.put(new KeyValPair<Long, String>(tuple.getTimestamp(), tuple.getKey()), new KeyValPair<Double, Boolean>(tuple.getAvgSpeed(), tuple.getSlowdownEvent()));
- }
- };
- }
-
- /**
- * Format the results of the slowdown calculations to a OutputPojo.
- */
- static class FormatStatsFn implements Function.MapFunction<Tuple.TimestampedTuple<KeyValPair<String, RouteInfo>>, OutputPojo>
- {
- @Override
- public OutputPojo f(Tuple.TimestampedTuple<KeyValPair<String, RouteInfo>> input)
- {
- RouteInfo routeInfo = input.getValue().getValue();
- OutputPojo row = new OutputPojo(routeInfo.getAvgSpeed(), routeInfo.getSlowdownEvent(), input.getValue().getKey(), input.getTimestamp());
- return row;
- }
- }
-
-
- /**
- * This composite transformation extracts speed info from traffic station readings.
- * It groups the readings by 'route' and analyzes traffic slowdown for that route.
- * Lastly, it formats the results for JDBC.
- */
- static class TrackSpeed extends
- CompositeStreamTransform<WindowedStream<KeyValPair<String, StationSpeed>>, WindowedStream<OutputPojo>>
- {
- @Override
- public WindowedStream<OutputPojo> compose(WindowedStream<KeyValPair<String, StationSpeed>> inputStream)
- {
- // Apply a GroupByKey transform to collect a list of all station
- // readings for a given route.
- WindowedStream<Tuple.WindowedTuple<KeyValPair<String, List<StationSpeed>>>> timeGroup =
- inputStream
- .accumulateByKey(new Group<StationSpeed>(), new Function.ToKeyValue<KeyValPair<String, StationSpeed>, String, StationSpeed>()
- {
- @Override
- public Tuple<KeyValPair<String, StationSpeed>> f(KeyValPair<String, StationSpeed> input)
- {
- return new Tuple.TimestampedTuple<>(input.getValue().getTimestamp(), input);
- }
- }, name("GroupByKey"));
-
- // Analyze 'slowdown' over the route readings.
- WindowedStream<Tuple.TimestampedTuple<KeyValPair<String, RouteInfo>>> stats = timeGroup
- .flatMap(new GatherStats(), name("GatherStats"));
-
- // Format the results for writing to JDBC table.
- WindowedStream<OutputPojo> results = stats.map(new FormatStatsFn(), name("FormatStatsFn"));
-
- return results;
- }
- }
-
-
- private static Double tryParseAvgSpeed(String[] inputItems)
- {
- try {
- return Double.parseDouble(tryParseString(inputItems, 3));
- } catch (NumberFormatException e) {
- return null;
- } catch (NullPointerException e) {
- return null;
- }
- }
-
- private static String tryParseStationType(String[] inputItems)
- {
- return tryParseString(inputItems, 2);
- }
-
- private static String tryParseStationId(String[] inputItems)
- {
- return tryParseString(inputItems, 1);
- }
-
- private static String tryParseTimestamp(String[] inputItems)
- {
- return tryParseString(inputItems, 0);
- }
-
- private static String tryParseString(String[] inputItems, int index)
- {
- return inputItems.length >= index ? inputItems[index] : null;
- }
-
- /**
- * Define some small hard-wired San Diego 'routes' to track based on sensor station ID.
- */
- private static Map<String, String> buildStationInfo()
- {
- Map<String, String> stations = new Hashtable<String, String>();
- stations.put("1108413", "SDRoute1"); // from freeway 805 S
- stations.put("1108699", "SDRoute2"); // from freeway 78 E
- stations.put("1108702", "SDRoute2");
- return stations;
- }
-
- /**
- * A dummy generator to generate some traffic information.
- */
- public static class InfoGen extends BaseOperator implements InputOperator
- {
- public transient DefaultOutputPort<String> output = new DefaultOutputPort<>();
-
- private String[] stationTypes = new String[]{"ML", "BL", "GL"};
- private int[] stationIDs = new int[]{1108413, 1108699, 1108702};
- private double ave = 55.0;
- private long timestamp;
- private static final Duration RAND_RANGE = Duration.standardMinutes(10);
- private static int tupleCount = 0;
-
- public static int getTupleCount()
- {
- return tupleCount;
- }
-
- @Override
- public void setup(Context.OperatorContext context)
- {
- tupleCount = 0;
- timestamp = System.currentTimeMillis();
- }
-
- @Override
- public void emitTuples()
- {
- for (String stationType : stationTypes) {
- for (int stationID : stationIDs) {
- double speed = Math.random() * 20 + ave;
- long time = (long)(Math.random() * RAND_RANGE.getMillis()) + timestamp;
- try {
- output.emit(time + "," + stationID + "," + stationType + "," + speed);
- tupleCount++;
-
- Thread.sleep(50);
- } catch (Exception e) {
- // Ignore it
- }
- }
- }
- }
- }
-
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- InfoGen infoGen = new InfoGen();
- Collector collector = new Collector();
-
- // Create a stream from the input operator.
- ApexStream<Tuple.TimestampedTuple<String>> stream = StreamFactory.fromInput(infoGen, infoGen.output, name("infoGen"))
-
- // Extract the timestamp from the input and wrap it into a TimestampedTuple.
- .map(new ExtractTimestamps(), name("ExtractTimestamps"));
-
- stream
- // Extract the average speed of a station.
- .flatMap(new ExtractStationSpeedFn(), name("ExtractStationSpeedFn"))
-
- // Apply window and trigger option.
- .window(new WindowOption.SlidingTimeWindows(Duration.standardMinutes(WINDOW_DURATION), Duration.standardMinutes(WINDOW_SLIDE_EVERY)), new TriggerOption().withEarlyFiringsAtEvery(Duration.millis(5000)).accumulatingFiredPanes())
-
- // Apply TrackSpeed composite transformation to compute the route information.
- .addCompositeStreams(new TrackSpeed())
-
- // print the result to console.
- .print(name("console"))
- .endWith(collector, collector.input, name("Collector"))
- .populateDag(dag);
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/TwitterAutoComplete.java
----------------------------------------------------------------------
diff --git a/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/TwitterAutoComplete.java b/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/TwitterAutoComplete.java
deleted file mode 100644
index 4fc80ea..0000000
--- a/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/complete/TwitterAutoComplete.java
+++ /dev/null
@@ -1,254 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.apex.malhar.stream.sample.complete;
-
-import java.nio.charset.Charset;
-import java.nio.charset.CharsetEncoder;
-
-import java.util.LinkedList;
-import java.util.List;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.joda.time.Duration;
-
-import org.apache.apex.malhar.lib.window.TriggerOption;
-import org.apache.apex.malhar.lib.window.Tuple;
-import org.apache.apex.malhar.lib.window.Window;
-import org.apache.apex.malhar.lib.window.WindowOption;
-
-import org.apache.apex.malhar.stream.api.ApexStream;
-import org.apache.apex.malhar.stream.api.CompositeStreamTransform;
-import org.apache.apex.malhar.stream.api.WindowedStream;
-import org.apache.apex.malhar.stream.api.function.Function;
-import org.apache.apex.malhar.stream.api.impl.StreamFactory;
-
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-import com.datatorrent.contrib.twitter.TwitterSampleInput;
-import com.datatorrent.lib.util.KeyValPair;
-
-import static org.apache.apex.malhar.stream.api.Option.Options.name;
-
-/**
- * Auto Complete Hashtag Demo with real time twitter input. In order to run this application, you need to create an app
- * at https://apps.twitter.com, then generate your consumer and access keys and tokens, and enter those information
- * accordingly in /resources/META-INF/properties.xml.
- *
- * The authentication requires following 4 information.
- * Your application consumer key,
- * Your application consumer secret,
- * Your twitter access token, and
- * Your twitter access token secret.
- *
- * @since 3.5.0
- */
-@ApplicationAnnotation(name = "TwitterAutoComplete")
-public class TwitterAutoComplete implements StreamingApplication
-{
- /**
- * Check whether every character in a string is ASCII encoding.
- */
- public static class StringUtils
- {
- static CharsetEncoder encoder = Charset.forName("US-ASCII").newEncoder();
-
- public static boolean isAscii(String v)
- {
- return encoder.canEncode(v);
- }
- }
-
- /**
- * FlapMap Function to extract all hashtags from a text form tweet.
- */
- private static class ExtractHashtags implements Function.FlatMapFunction<String, String>
- {
-
- @Override
- public Iterable<String> f(String input)
- {
- List<String> result = new LinkedList<>();
- Matcher m = Pattern.compile("#\\S+").matcher(input);
- while (m.find()) {
- result.add(m.group().substring(1));
- }
- return result;
- }
- }
-
- /**
- * Lower latency, but more expensive.
- */
- private static class ComputeTopFlat
- extends CompositeStreamTransform<WindowedStream<CompletionCandidate>, WindowedStream<Tuple.WindowedTuple<KeyValPair<String, List<CompletionCandidate>>>>>
- {
- private final int candidatesPerPrefix;
- private final int minPrefix;
-
- public ComputeTopFlat(int candidatesPerPrefix, int minPrefix)
- {
- this.candidatesPerPrefix = candidatesPerPrefix;
- this.minPrefix = minPrefix;
- }
-
- @Override
- public WindowedStream<Tuple.WindowedTuple<KeyValPair<String, List<CompletionCandidate>>>> compose(
- WindowedStream<CompletionCandidate> input)
- {
- TopNByKey topNByKey = new TopNByKey();
- topNByKey.setN(candidatesPerPrefix);
- return input
- .<KeyValPair<String, CompletionCandidate>, WindowedStream<KeyValPair<String, CompletionCandidate>>>flatMap(new AllPrefixes(minPrefix, 3), name("Extract Prefixes"))
- .accumulateByKey(topNByKey, new Function.ToKeyValue<KeyValPair<String, CompletionCandidate>, String, CompletionCandidate>()
- {
- @Override
- public Tuple<KeyValPair<String, CompletionCandidate>> f(KeyValPair<String, CompletionCandidate> tuple)
- {
- // TODO: Should be removed after Auto-wrapping is supported.
- return new Tuple.WindowedTuple<>(Window.GlobalWindow.INSTANCE, tuple);
- }
- }, name("TopNByKey"));
- }
- }
-
- /**
- * FlapMap Function to extract all prefixes of the hashtag in the input CompletionCandidate, and output
- * KeyValPairs of the prefix and the CompletionCandidate
- */
- private static class AllPrefixes implements Function.FlatMapFunction<CompletionCandidate, KeyValPair<String, CompletionCandidate>>
- {
- private final int minPrefix;
- private final int maxPrefix;
-
- public AllPrefixes()
- {
- this(0, Integer.MAX_VALUE);
- }
-
- public AllPrefixes(int minPrefix)
- {
- this(minPrefix, Integer.MAX_VALUE);
- }
-
- public AllPrefixes(int minPrefix, int maxPrefix)
- {
- this.minPrefix = minPrefix;
- this.maxPrefix = maxPrefix;
- }
-
- @Override
- public Iterable<KeyValPair<String, CompletionCandidate>> f(CompletionCandidate input)
- {
- List<KeyValPair<String, CompletionCandidate>> result = new LinkedList<>();
- String word = input.getValue();
- for (int i = minPrefix; i <= Math.min(word.length(), maxPrefix); i++) {
- result.add(new KeyValPair<>(input.getValue().substring(0, i).toLowerCase(), input));
- }
- return result;
- }
- }
-
- /**
- * A Composite stream transform that takes as input a list of tokens and returns
- * the most common tokens per prefix.
- */
- public static class ComputeTopCompletions
- extends CompositeStreamTransform<WindowedStream<String>, WindowedStream<Tuple.WindowedTuple<KeyValPair<String, List<CompletionCandidate>>>>>
- {
- private final int candidatesPerPrefix;
- private final boolean recursive;
-
- protected ComputeTopCompletions(int candidatesPerPrefix, boolean recursive)
- {
- this.candidatesPerPrefix = candidatesPerPrefix;
- this.recursive = recursive;
- }
-
- public static ComputeTopCompletions top(int candidatesPerPrefix, boolean recursive)
- {
- return new ComputeTopCompletions(candidatesPerPrefix, recursive);
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public WindowedStream<Tuple.WindowedTuple<KeyValPair<String, List<CompletionCandidate>>>> compose(WindowedStream<String> inputStream)
- {
-
- ApexStream<CompletionCandidate> candidates = inputStream
- .countByKey(new Function.ToKeyValue<String, String, Long>()
- {
- @Override
- public Tuple<KeyValPair<String, Long>> f(String input)
- {
- return new Tuple.PlainTuple<>(new KeyValPair<>(input, 1L));
- }
- }, name("Hashtag Count"))
- .map(new Function.MapFunction<Tuple.WindowedTuple<KeyValPair<String,Long>>, CompletionCandidate>()
- {
- @Override
- public CompletionCandidate f(Tuple.WindowedTuple<KeyValPair<String, Long>> input)
- {
- return new CompletionCandidate(input.getValue().getKey(), input.getValue().getValue());
- }
- }, name("KeyValPair to CompletionCandidate"));
-
- return candidates.addCompositeStreams(new ComputeTopFlat(candidatesPerPrefix, 1));
-
- }
- }
-
- /**
- * FilterFunction to filter out tweets with non-acsii characters.
- */
- static class ASCIIFilter implements Function.FilterFunction<String>
- {
- @Override
- public boolean f(String input)
- {
- return StringUtils.isAscii(input);
- }
- }
-
- /**
- * Populate the dag with High-Level API.
- * @param dag
- * @param conf
- */
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- TwitterSampleInput input = new TwitterSampleInput();
-
- WindowOption windowOption = new WindowOption.GlobalWindow();
-
- ApexStream<String> tags = StreamFactory.fromInput(input, input.text, name("tweetSampler"))
- .filter(new ASCIIFilter(), name("ACSII Filter"))
- .flatMap(new ExtractHashtags(), name("Extract Hashtags"));
-
- ApexStream<Tuple.WindowedTuple<KeyValPair<String, List<CompletionCandidate>>>> s =
- tags.window(windowOption, new TriggerOption().accumulatingFiredPanes().withEarlyFiringsAtEvery(Duration.standardSeconds(10)))
- .addCompositeStreams(ComputeTopCompletions.top(10, true)).print();
-
- s.populateDag(dag);
- }
-}
[24/30] apex-malhar git commit: Renamed demos to examples. Packages
and artifactid names are changed as suggested.
Posted by th...@apache.org.
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/machinedata/src/main/html/malhar.css
----------------------------------------------------------------------
diff --git a/demos/machinedata/src/main/html/malhar.css b/demos/machinedata/src/main/html/malhar.css
deleted file mode 100644
index 175e219..0000000
--- a/demos/machinedata/src/main/html/malhar.css
+++ /dev/null
@@ -1,4688 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-.clearfix {
- *zoom: 1;
-}
-.clearfix:before,
-.clearfix:after {
- display: table;
- content: "";
- line-height: 0;
-}
-.clearfix:after {
- clear: both;
-}
-.hide-text {
- font: 0/0 a;
- color: transparent;
- text-shadow: none;
- background-color: transparent;
- border: 0;
-}
-.input-block-level {
- display: block;
- width: 100%;
- min-height: 30px;
- -webkit-box-sizing: border-box;
- -moz-box-sizing: border-box;
- box-sizing: border-box;
-}
-article,
-aside,
-details,
-figcaption,
-figure,
-footer,
-header,
-hgroup,
-nav,
-section {
- display: block;
-}
-audio,
-canvas,
-video {
- display: inline-block;
- *display: inline;
- *zoom: 1;
-}
-audio:not([controls]) {
- display: none;
-}
-html {
- font-size: 100%;
- -webkit-text-size-adjust: 100%;
- -ms-text-size-adjust: 100%;
-}
-a:focus {
- outline: thin dotted #333;
- outline: 5px auto -webkit-focus-ring-color;
- outline-offset: -2px;
-}
-a:hover,
-a:active {
- outline: 0;
-}
-sub,
-sup {
- position: relative;
- font-size: 75%;
- line-height: 0;
- vertical-align: baseline;
-}
-sup {
- top: -0.5em;
-}
-sub {
- bottom: -0.25em;
-}
-img {
- /* Responsive images (ensure images don't scale beyond their parents) */
-
- max-width: 100%;
- /* Part 1: Set a maxium relative to the parent */
-
- width: auto\9;
- /* IE7-8 need help adjusting responsive images */
-
- height: auto;
- /* Part 2: Scale the height according to the width, otherwise you get stretching */
-
- vertical-align: middle;
- border: 0;
- -ms-interpolation-mode: bicubic;
-}
-#map_canvas img,
-.google-maps img {
- max-width: none;
-}
-button,
-input,
-select,
-textarea {
- margin: 0;
- font-size: 100%;
- vertical-align: middle;
-}
-button,
-input {
- *overflow: visible;
- line-height: normal;
-}
-button::-moz-focus-inner,
-input::-moz-focus-inner {
- padding: 0;
- border: 0;
-}
-button,
-html input[type="button"],
-input[type="reset"],
-input[type="submit"] {
- -webkit-appearance: button;
- cursor: pointer;
-}
-label,
-select,
-button,
-input[type="button"],
-input[type="reset"],
-input[type="submit"],
-input[type="radio"],
-input[type="checkbox"] {
- cursor: pointer;
-}
-input[type="search"] {
- -webkit-box-sizing: content-box;
- -moz-box-sizing: content-box;
- box-sizing: content-box;
- -webkit-appearance: textfield;
-}
-input[type="search"]::-webkit-search-decoration,
-input[type="search"]::-webkit-search-cancel-button {
- -webkit-appearance: none;
-}
-textarea {
- overflow: auto;
- vertical-align: top;
-}
-@media print {
- * {
- text-shadow: none !important;
- color: #000 !important;
- background: transparent !important;
- box-shadow: none !important;
- }
- a,
- a:visited {
- text-decoration: underline;
- }
- a[href]:after {
- content: " (" attr(href) ")";
- }
- abbr[title]:after {
- content: " (" attr(title) ")";
- }
- .ir a:after,
- a[href^="javascript:"]:after,
- a[href^="#"]:after {
- content: "";
- }
- pre,
- blockquote {
- border: 1px solid #999;
- page-break-inside: avoid;
- }
- thead {
- display: table-header-group;
- }
- tr,
- img {
- page-break-inside: avoid;
- }
- img {
- max-width: 100% !important;
- }
- @page {
- margin: 0.5cm;
- }
- p,
- h2,
- h3 {
- orphans: 3;
- widows: 3;
- }
- h2,
- h3 {
- page-break-after: avoid;
- }
-}
-body {
- margin: 0;
- font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
- font-size: 13px;
- line-height: 20px;
- color: #333333;
- background-color: #ffffff;
-}
-a {
- color: #0088cc;
- text-decoration: none;
-}
-a:hover,
-a:focus {
- color: #005580;
- text-decoration: underline;
-}
-.img-rounded {
- -webkit-border-radius: 6px;
- -moz-border-radius: 6px;
- border-radius: 6px;
-}
-.img-polaroid {
- padding: 4px;
- background-color: #fff;
- border: 1px solid #ccc;
- border: 1px solid rgba(0, 0, 0, 0.2);
- -webkit-box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1);
- -moz-box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1);
- box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1);
-}
-.img-circle {
- -webkit-border-radius: 500px;
- -moz-border-radius: 500px;
- border-radius: 500px;
-}
-/*@import "bootstrap/grid.less";*/
-/*@import "bootstrap/layouts.less";*/
-p {
- margin: 0 0 10px;
-}
-.lead {
- margin-bottom: 20px;
- font-size: 19.5px;
- font-weight: 200;
- line-height: 30px;
-}
-small {
- font-size: 85%;
-}
-strong {
- font-weight: bold;
-}
-em {
- font-style: italic;
-}
-cite {
- font-style: normal;
-}
-.muted {
- color: #999999;
-}
-a.muted:hover,
-a.muted:focus {
- color: #808080;
-}
-.text-warning {
- color: #c09853;
-}
-a.text-warning:hover,
-a.text-warning:focus {
- color: #a47e3c;
-}
-.text-error {
- color: #b94a48;
-}
-a.text-error:hover,
-a.text-error:focus {
- color: #953b39;
-}
-.text-info {
- color: #3a87ad;
-}
-a.text-info:hover,
-a.text-info:focus {
- color: #2d6987;
-}
-.text-success {
- color: #468847;
-}
-a.text-success:hover,
-a.text-success:focus {
- color: #356635;
-}
-.text-left {
- text-align: left;
-}
-.text-right {
- text-align: right;
-}
-.text-center {
- text-align: center;
-}
-h1,
-h2,
-h3,
-h4,
-h5,
-h6 {
- margin: 10px 0;
- font-family: inherit;
- font-weight: bold;
- line-height: 20px;
- color: inherit;
- text-rendering: optimizelegibility;
-}
-h1 small,
-h2 small,
-h3 small,
-h4 small,
-h5 small,
-h6 small {
- font-weight: normal;
- line-height: 1;
- color: #999999;
-}
-h1,
-h2,
-h3 {
- line-height: 40px;
-}
-h1 {
- font-size: 35.75px;
-}
-h2 {
- font-size: 29.25px;
-}
-h3 {
- font-size: 22.75px;
-}
-h4 {
- font-size: 16.25px;
-}
-h5 {
- font-size: 13px;
-}
-h6 {
- font-size: 11.049999999999999px;
-}
-h1 small {
- font-size: 22.75px;
-}
-h2 small {
- font-size: 16.25px;
-}
-h3 small {
- font-size: 13px;
-}
-h4 small {
- font-size: 13px;
-}
-.page-header {
- padding-bottom: 9px;
- margin: 20px 0 30px;
- border-bottom: 1px solid #eeeeee;
-}
-ul,
-ol {
- padding: 0;
- margin: 0 0 10px 25px;
-}
-ul ul,
-ul ol,
-ol ol,
-ol ul {
- margin-bottom: 0;
-}
-li {
- line-height: 20px;
-}
-ul.unstyled,
-ol.unstyled {
- margin-left: 0;
- list-style: none;
-}
-ul.inline,
-ol.inline {
- margin-left: 0;
- list-style: none;
-}
-ul.inline > li,
-ol.inline > li {
- display: inline-block;
- *display: inline;
- /* IE7 inline-block hack */
-
- *zoom: 1;
- padding-left: 5px;
- padding-right: 5px;
-}
-dl {
- margin-bottom: 20px;
-}
-dt,
-dd {
- line-height: 20px;
-}
-dt {
- font-weight: bold;
-}
-dd {
- margin-left: 10px;
-}
-.dl-horizontal {
- *zoom: 1;
-}
-.dl-horizontal:before,
-.dl-horizontal:after {
- display: table;
- content: "";
- line-height: 0;
-}
-.dl-horizontal:after {
- clear: both;
-}
-.dl-horizontal dt {
- float: left;
- width: 160px;
- clear: left;
- text-align: right;
- overflow: hidden;
- text-overflow: ellipsis;
- white-space: nowrap;
-}
-.dl-horizontal dd {
- margin-left: 180px;
-}
-hr {
- margin: 20px 0;
- border: 0;
- border-top: 1px solid #eeeeee;
- border-bottom: 1px solid #ffffff;
-}
-abbr[title],
-abbr[data-original-title] {
- cursor: help;
- border-bottom: 1px dotted #999999;
-}
-abbr.initialism {
- font-size: 90%;
- text-transform: uppercase;
-}
-blockquote {
- padding: 0 0 0 15px;
- margin: 0 0 20px;
- border-left: 5px solid #eeeeee;
-}
-blockquote p {
- margin-bottom: 0;
- font-size: 16.25px;
- font-weight: 300;
- line-height: 1.25;
-}
-blockquote small {
- display: block;
- line-height: 20px;
- color: #999999;
-}
-blockquote small:before {
- content: '\2014 \00A0';
-}
-blockquote.pull-right {
- float: right;
- padding-right: 15px;
- padding-left: 0;
- border-right: 5px solid #eeeeee;
- border-left: 0;
-}
-blockquote.pull-right p,
-blockquote.pull-right small {
- text-align: right;
-}
-blockquote.pull-right small:before {
- content: '';
-}
-blockquote.pull-right small:after {
- content: '\00A0 \2014';
-}
-q:before,
-q:after,
-blockquote:before,
-blockquote:after {
- content: "";
-}
-address {
- display: block;
- margin-bottom: 20px;
- font-style: normal;
- line-height: 20px;
-}
-code,
-pre {
- padding: 0 3px 2px;
- font-family: Monaco, Menlo, Consolas, "Courier New", monospace;
- font-size: 11px;
- color: #333333;
- -webkit-border-radius: 3px;
- -moz-border-radius: 3px;
- border-radius: 3px;
-}
-code {
- padding: 2px 4px;
- color: #d14;
- background-color: #f7f7f9;
- border: 1px solid #e1e1e8;
- white-space: nowrap;
-}
-pre {
- display: block;
- padding: 9.5px;
- margin: 0 0 10px;
- font-size: 12px;
- line-height: 20px;
- word-break: break-all;
- word-wrap: break-word;
- white-space: pre;
- white-space: pre-wrap;
- background-color: #f5f5f5;
- border: 1px solid #ccc;
- border: 1px solid rgba(0, 0, 0, 0.15);
- -webkit-border-radius: 4px;
- -moz-border-radius: 4px;
- border-radius: 4px;
-}
-pre.prettyprint {
- margin-bottom: 20px;
-}
-pre code {
- padding: 0;
- color: inherit;
- white-space: pre;
- white-space: pre-wrap;
- background-color: transparent;
- border: 0;
-}
-.pre-scrollable {
- max-height: 340px;
- overflow-y: scroll;
-}
-form {
- margin: 0 0 20px;
-}
-fieldset {
- padding: 0;
- margin: 0;
- border: 0;
-}
-legend {
- display: block;
- width: 100%;
- padding: 0;
- margin-bottom: 20px;
- font-size: 19.5px;
- line-height: 40px;
- color: #333333;
- border: 0;
- border-bottom: 1px solid #e5e5e5;
-}
-legend small {
- font-size: 15px;
- color: #999999;
-}
-label,
-input,
-button,
-select,
-textarea {
- font-size: 13px;
- font-weight: normal;
- line-height: 20px;
-}
-input,
-button,
-select,
-textarea {
- font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
-}
-label {
- display: block;
- margin-bottom: 5px;
-}
-select,
-textarea,
-input[type="text"],
-input[type="password"],
-input[type="datetime"],
-input[type="datetime-local"],
-input[type="date"],
-input[type="month"],
-input[type="time"],
-input[type="week"],
-input[type="number"],
-input[type="email"],
-input[type="url"],
-input[type="search"],
-input[type="tel"],
-input[type="color"],
-.uneditable-input {
- display: inline-block;
- height: 20px;
- padding: 4px 6px;
- margin-bottom: 10px;
- font-size: 13px;
- line-height: 20px;
- color: #555555;
- -webkit-border-radius: 4px;
- -moz-border-radius: 4px;
- border-radius: 4px;
- vertical-align: middle;
-}
-input,
-textarea,
-.uneditable-input {
- width: 206px;
-}
-textarea {
- height: auto;
-}
-textarea,
-input[type="text"],
-input[type="password"],
-input[type="datetime"],
-input[type="datetime-local"],
-input[type="date"],
-input[type="month"],
-input[type="time"],
-input[type="week"],
-input[type="number"],
-input[type="email"],
-input[type="url"],
-input[type="search"],
-input[type="tel"],
-input[type="color"],
-.uneditable-input {
- background-color: #ffffff;
- border: 1px solid #cccccc;
- -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
- -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
- box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
- -webkit-transition: border linear .2s, box-shadow linear .2s;
- -moz-transition: border linear .2s, box-shadow linear .2s;
- -o-transition: border linear .2s, box-shadow linear .2s;
- transition: border linear .2s, box-shadow linear .2s;
-}
-textarea:focus,
-input[type="text"]:focus,
-input[type="password"]:focus,
-input[type="datetime"]:focus,
-input[type="datetime-local"]:focus,
-input[type="date"]:focus,
-input[type="month"]:focus,
-input[type="time"]:focus,
-input[type="week"]:focus,
-input[type="number"]:focus,
-input[type="email"]:focus,
-input[type="url"]:focus,
-input[type="search"]:focus,
-input[type="tel"]:focus,
-input[type="color"]:focus,
-.uneditable-input:focus {
- border-color: rgba(82, 168, 236, 0.8);
- outline: 0;
- outline: thin dotted \9;
- /* IE6-9 */
-
- -webkit-box-shadow: inset 0 1px 1px rgba(0,0,0,.075), 0 0 8px rgba(82,168,236,.6);
- -moz-box-shadow: inset 0 1px 1px rgba(0,0,0,.075), 0 0 8px rgba(82,168,236,.6);
- box-shadow: inset 0 1px 1px rgba(0,0,0,.075), 0 0 8px rgba(82,168,236,.6);
-}
-input[type="radio"],
-input[type="checkbox"] {
- margin: 4px 0 0;
- *margin-top: 0;
- /* IE7 */
-
- margin-top: 1px \9;
- /* IE8-9 */
-
- line-height: normal;
-}
-input[type="file"],
-input[type="image"],
-input[type="submit"],
-input[type="reset"],
-input[type="button"],
-input[type="radio"],
-input[type="checkbox"] {
- width: auto;
-}
-select,
-input[type="file"] {
- height: 30px;
- /* In IE7, the height of the select element cannot be changed by height, only font-size */
-
- *margin-top: 4px;
- /* For IE7, add top margin to align select with labels */
-
- line-height: 30px;
-}
-select {
- width: 220px;
- border: 1px solid #cccccc;
- background-color: #ffffff;
-}
-select[multiple],
-select[size] {
- height: auto;
-}
-select:focus,
-input[type="file"]:focus,
-input[type="radio"]:focus,
-input[type="checkbox"]:focus {
- outline: thin dotted #333;
- outline: 5px auto -webkit-focus-ring-color;
- outline-offset: -2px;
-}
-.uneditable-input,
-.uneditable-textarea {
- color: #999999;
- background-color: #fcfcfc;
- border-color: #cccccc;
- -webkit-box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.025);
- -moz-box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.025);
- box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.025);
- cursor: not-allowed;
-}
-.uneditable-input {
- overflow: hidden;
- white-space: nowrap;
-}
-.uneditable-textarea {
- width: auto;
- height: auto;
-}
-input:-moz-placeholder,
-textarea:-moz-placeholder {
- color: #999999;
-}
-input:-ms-input-placeholder,
-textarea:-ms-input-placeholder {
- color: #999999;
-}
-input::-webkit-input-placeholder,
-textarea::-webkit-input-placeholder {
- color: #999999;
-}
-.radio,
-.checkbox {
- min-height: 20px;
- padding-left: 20px;
-}
-.radio input[type="radio"],
-.checkbox input[type="checkbox"] {
- float: left;
- margin-left: -20px;
-}
-.controls > .radio:first-child,
-.controls > .checkbox:first-child {
- padding-top: 5px;
-}
-.radio.inline,
-.checkbox.inline {
- display: inline-block;
- padding-top: 5px;
- margin-bottom: 0;
- vertical-align: middle;
-}
-.radio.inline + .radio.inline,
-.checkbox.inline + .checkbox.inline {
- margin-left: 10px;
-}
-.input-mini {
- width: 60px;
-}
-.input-small {
- width: 90px;
-}
-.input-medium {
- width: 150px;
-}
-.input-large {
- width: 210px;
-}
-.input-xlarge {
- width: 270px;
-}
-.input-xxlarge {
- width: 530px;
-}
-input[class*="span"],
-select[class*="span"],
-textarea[class*="span"],
-.uneditable-input[class*="span"],
-.row-fluid input[class*="span"],
-.row-fluid select[class*="span"],
-.row-fluid textarea[class*="span"],
-.row-fluid .uneditable-input[class*="span"] {
- float: none;
- margin-left: 0;
-}
-.input-append input[class*="span"],
-.input-append .uneditable-input[class*="span"],
-.input-prepend input[class*="span"],
-.input-prepend .uneditable-input[class*="span"],
-.row-fluid input[class*="span"],
-.row-fluid select[class*="span"],
-.row-fluid textarea[class*="span"],
-.row-fluid .uneditable-input[class*="span"],
-.row-fluid .input-prepend [class*="span"],
-.row-fluid .input-append [class*="span"] {
- display: inline-block;
-}
-input,
-textarea,
-.uneditable-input {
- margin-left: 0;
-}
-.controls-row [class*="span"] + [class*="span"] {
- margin-left: 20px;
-}
-input.span12,
-textarea.span12,
-.uneditable-input.span12 {
- width: 926px;
-}
-input.span11,
-textarea.span11,
-.uneditable-input.span11 {
- width: 846px;
-}
-input.span10,
-textarea.span10,
-.uneditable-input.span10 {
- width: 766px;
-}
-input.span9,
-textarea.span9,
-.uneditable-input.span9 {
- width: 686px;
-}
-input.span8,
-textarea.span8,
-.uneditable-input.span8 {
- width: 606px;
-}
-input.span7,
-textarea.span7,
-.uneditable-input.span7 {
- width: 526px;
-}
-input.span6,
-textarea.span6,
-.uneditable-input.span6 {
- width: 446px;
-}
-input.span5,
-textarea.span5,
-.uneditable-input.span5 {
- width: 366px;
-}
-input.span4,
-textarea.span4,
-.uneditable-input.span4 {
- width: 286px;
-}
-input.span3,
-textarea.span3,
-.uneditable-input.span3 {
- width: 206px;
-}
-input.span2,
-textarea.span2,
-.uneditable-input.span2 {
- width: 126px;
-}
-input.span1,
-textarea.span1,
-.uneditable-input.span1 {
- width: 46px;
-}
-.controls-row {
- *zoom: 1;
-}
-.controls-row:before,
-.controls-row:after {
- display: table;
- content: "";
- line-height: 0;
-}
-.controls-row:after {
- clear: both;
-}
-.controls-row [class*="span"],
-.row-fluid .controls-row [class*="span"] {
- float: left;
-}
-.controls-row .checkbox[class*="span"],
-.controls-row .radio[class*="span"] {
- padding-top: 5px;
-}
-input[disabled],
-select[disabled],
-textarea[disabled],
-input[readonly],
-select[readonly],
-textarea[readonly] {
- cursor: not-allowed;
- background-color: #eeeeee;
-}
-input[type="radio"][disabled],
-input[type="checkbox"][disabled],
-input[type="radio"][readonly],
-input[type="checkbox"][readonly] {
- background-color: transparent;
-}
-.control-group.warning .control-label,
-.control-group.warning .help-block,
-.control-group.warning .help-inline {
- color: #c09853;
-}
-.control-group.warning .checkbox,
-.control-group.warning .radio,
-.control-group.warning input,
-.control-group.warning select,
-.control-group.warning textarea {
- color: #c09853;
-}
-.control-group.warning input,
-.control-group.warning select,
-.control-group.warning textarea {
- border-color: #c09853;
- -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
- -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
- box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
-}
-.control-group.warning input:focus,
-.control-group.warning select:focus,
-.control-group.warning textarea:focus {
- border-color: #a47e3c;
- -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #dbc59e;
- -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #dbc59e;
- box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #dbc59e;
-}
-.control-group.warning .input-prepend .add-on,
-.control-group.warning .input-append .add-on {
- color: #c09853;
- background-color: #fcf8e3;
- border-color: #c09853;
-}
-.control-group.error .control-label,
-.control-group.error .help-block,
-.control-group.error .help-inline {
- color: #b94a48;
-}
-.control-group.error .checkbox,
-.control-group.error .radio,
-.control-group.error input,
-.control-group.error select,
-.control-group.error textarea {
- color: #b94a48;
-}
-.control-group.error input,
-.control-group.error select,
-.control-group.error textarea {
- border-color: #b94a48;
- -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
- -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
- box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
-}
-.control-group.error input:focus,
-.control-group.error select:focus,
-.control-group.error textarea:focus {
- border-color: #953b39;
- -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #d59392;
- -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #d59392;
- box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #d59392;
-}
-.control-group.error .input-prepend .add-on,
-.control-group.error .input-append .add-on {
- color: #b94a48;
- background-color: #f2dede;
- border-color: #b94a48;
-}
-.control-group.success .control-label,
-.control-group.success .help-block,
-.control-group.success .help-inline {
- color: #468847;
-}
-.control-group.success .checkbox,
-.control-group.success .radio,
-.control-group.success input,
-.control-group.success select,
-.control-group.success textarea {
- color: #468847;
-}
-.control-group.success input,
-.control-group.success select,
-.control-group.success textarea {
- border-color: #468847;
- -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
- -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
- box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
-}
-.control-group.success input:focus,
-.control-group.success select:focus,
-.control-group.success textarea:focus {
- border-color: #356635;
- -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #7aba7b;
- -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #7aba7b;
- box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #7aba7b;
-}
-.control-group.success .input-prepend .add-on,
-.control-group.success .input-append .add-on {
- color: #468847;
- background-color: #dff0d8;
- border-color: #468847;
-}
-.control-group.info .control-label,
-.control-group.info .help-block,
-.control-group.info .help-inline {
- color: #3a87ad;
-}
-.control-group.info .checkbox,
-.control-group.info .radio,
-.control-group.info input,
-.control-group.info select,
-.control-group.info textarea {
- color: #3a87ad;
-}
-.control-group.info input,
-.control-group.info select,
-.control-group.info textarea {
- border-color: #3a87ad;
- -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
- -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
- box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
-}
-.control-group.info input:focus,
-.control-group.info select:focus,
-.control-group.info textarea:focus {
- border-color: #2d6987;
- -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #7ab5d3;
- -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #7ab5d3;
- box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #7ab5d3;
-}
-.control-group.info .input-prepend .add-on,
-.control-group.info .input-append .add-on {
- color: #3a87ad;
- background-color: #d9edf7;
- border-color: #3a87ad;
-}
-input:focus:invalid,
-textarea:focus:invalid,
-select:focus:invalid {
- color: #b94a48;
- border-color: #ee5f5b;
-}
-input:focus:invalid:focus,
-textarea:focus:invalid:focus,
-select:focus:invalid:focus {
- border-color: #e9322d;
- -webkit-box-shadow: 0 0 6px #f8b9b7;
- -moz-box-shadow: 0 0 6px #f8b9b7;
- box-shadow: 0 0 6px #f8b9b7;
-}
-.form-actions {
- padding: 19px 20px 20px;
- margin-top: 20px;
- margin-bottom: 20px;
- background-color: #f5f5f5;
- border-top: 1px solid #e5e5e5;
- *zoom: 1;
-}
-.form-actions:before,
-.form-actions:after {
- display: table;
- content: "";
- line-height: 0;
-}
-.form-actions:after {
- clear: both;
-}
-.help-block,
-.help-inline {
- color: #595959;
-}
-.help-block {
- display: block;
- margin-bottom: 10px;
-}
-.help-inline {
- display: inline-block;
- *display: inline;
- /* IE7 inline-block hack */
-
- *zoom: 1;
- vertical-align: middle;
- padding-left: 5px;
-}
-.input-append,
-.input-prepend {
- display: inline-block;
- margin-bottom: 10px;
- vertical-align: middle;
- font-size: 0;
- white-space: nowrap;
-}
-.input-append input,
-.input-prepend input,
-.input-append select,
-.input-prepend select,
-.input-append .uneditable-input,
-.input-prepend .uneditable-input,
-.input-append .dropdown-menu,
-.input-prepend .dropdown-menu,
-.input-append .popover,
-.input-prepend .popover {
- font-size: 13px;
-}
-.input-append input,
-.input-prepend input,
-.input-append select,
-.input-prepend select,
-.input-append .uneditable-input,
-.input-prepend .uneditable-input {
- position: relative;
- margin-bottom: 0;
- *margin-left: 0;
- vertical-align: top;
- -webkit-border-radius: 0 4px 4px 0;
- -moz-border-radius: 0 4px 4px 0;
- border-radius: 0 4px 4px 0;
-}
-.input-append input:focus,
-.input-prepend input:focus,
-.input-append select:focus,
-.input-prepend select:focus,
-.input-append .uneditable-input:focus,
-.input-prepend .uneditable-input:focus {
- z-index: 2;
-}
-.input-append .add-on,
-.input-prepend .add-on {
- display: inline-block;
- width: auto;
- height: 20px;
- min-width: 16px;
- padding: 4px 5px;
- font-size: 13px;
- font-weight: normal;
- line-height: 20px;
- text-align: center;
- text-shadow: 0 1px 0 #ffffff;
- background-color: #eeeeee;
- border: 1px solid #ccc;
-}
-.input-append .add-on,
-.input-prepend .add-on,
-.input-append .btn,
-.input-prepend .btn,
-.input-append .btn-group > .dropdown-toggle,
-.input-prepend .btn-group > .dropdown-toggle {
- vertical-align: top;
- -webkit-border-radius: 0;
- -moz-border-radius: 0;
- border-radius: 0;
-}
-.input-append .active,
-.input-prepend .active {
- background-color: #a9dba9;
- border-color: #46a546;
-}
-.input-prepend .add-on,
-.input-prepend .btn {
- margin-right: -1px;
-}
-.input-prepend .add-on:first-child,
-.input-prepend .btn:first-child {
- -webkit-border-radius: 4px 0 0 4px;
- -moz-border-radius: 4px 0 0 4px;
- border-radius: 4px 0 0 4px;
-}
-.input-append input,
-.input-append select,
-.input-append .uneditable-input {
- -webkit-border-radius: 4px 0 0 4px;
- -moz-border-radius: 4px 0 0 4px;
- border-radius: 4px 0 0 4px;
-}
-.input-append input + .btn-group .btn:last-child,
-.input-append select + .btn-group .btn:last-child,
-.input-append .uneditable-input + .btn-group .btn:last-child {
- -webkit-border-radius: 0 4px 4px 0;
- -moz-border-radius: 0 4px 4px 0;
- border-radius: 0 4px 4px 0;
-}
-.input-append .add-on,
-.input-append .btn,
-.input-append .btn-group {
- margin-left: -1px;
-}
-.input-append .add-on:last-child,
-.input-append .btn:last-child,
-.input-append .btn-group:last-child > .dropdown-toggle {
- -webkit-border-radius: 0 4px 4px 0;
- -moz-border-radius: 0 4px 4px 0;
- border-radius: 0 4px 4px 0;
-}
-.input-prepend.input-append input,
-.input-prepend.input-append select,
-.input-prepend.input-append .uneditable-input {
- -webkit-border-radius: 0;
- -moz-border-radius: 0;
- border-radius: 0;
-}
-.input-prepend.input-append input + .btn-group .btn,
-.input-prepend.input-append select + .btn-group .btn,
-.input-prepend.input-append .uneditable-input + .btn-group .btn {
- -webkit-border-radius: 0 4px 4px 0;
- -moz-border-radius: 0 4px 4px 0;
- border-radius: 0 4px 4px 0;
-}
-.input-prepend.input-append .add-on:first-child,
-.input-prepend.input-append .btn:first-child {
- margin-right: -1px;
- -webkit-border-radius: 4px 0 0 4px;
- -moz-border-radius: 4px 0 0 4px;
- border-radius: 4px 0 0 4px;
-}
-.input-prepend.input-append .add-on:last-child,
-.input-prepend.input-append .btn:last-child {
- margin-left: -1px;
- -webkit-border-radius: 0 4px 4px 0;
- -moz-border-radius: 0 4px 4px 0;
- border-radius: 0 4px 4px 0;
-}
-.input-prepend.input-append .btn-group:first-child {
- margin-left: 0;
-}
-input.search-query {
- padding-right: 14px;
- padding-right: 4px \9;
- padding-left: 14px;
- padding-left: 4px \9;
- /* IE7-8 doesn't have border-radius, so don't indent the padding */
-
- margin-bottom: 0;
- -webkit-border-radius: 15px;
- -moz-border-radius: 15px;
- border-radius: 15px;
-}
-/* Allow for input prepend/append in search forms */
-.form-search .input-append .search-query,
-.form-search .input-prepend .search-query {
- -webkit-border-radius: 0;
- -moz-border-radius: 0;
- border-radius: 0;
-}
-.form-search .input-append .search-query {
- -webkit-border-radius: 14px 0 0 14px;
- -moz-border-radius: 14px 0 0 14px;
- border-radius: 14px 0 0 14px;
-}
-.form-search .input-append .btn {
- -webkit-border-radius: 0 14px 14px 0;
- -moz-border-radius: 0 14px 14px 0;
- border-radius: 0 14px 14px 0;
-}
-.form-search .input-prepend .search-query {
- -webkit-border-radius: 0 14px 14px 0;
- -moz-border-radius: 0 14px 14px 0;
- border-radius: 0 14px 14px 0;
-}
-.form-search .input-prepend .btn {
- -webkit-border-radius: 14px 0 0 14px;
- -moz-border-radius: 14px 0 0 14px;
- border-radius: 14px 0 0 14px;
-}
-.form-search input,
-.form-inline input,
-.form-horizontal input,
-.form-search textarea,
-.form-inline textarea,
-.form-horizontal textarea,
-.form-search select,
-.form-inline select,
-.form-horizontal select,
-.form-search .help-inline,
-.form-inline .help-inline,
-.form-horizontal .help-inline,
-.form-search .uneditable-input,
-.form-inline .uneditable-input,
-.form-horizontal .uneditable-input,
-.form-search .input-prepend,
-.form-inline .input-prepend,
-.form-horizontal .input-prepend,
-.form-search .input-append,
-.form-inline .input-append,
-.form-horizontal .input-append {
- display: inline-block;
- *display: inline;
- /* IE7 inline-block hack */
-
- *zoom: 1;
- margin-bottom: 0;
- vertical-align: middle;
-}
-.form-search .hide,
-.form-inline .hide,
-.form-horizontal .hide {
- display: none;
-}
-.form-search label,
-.form-inline label,
-.form-search .btn-group,
-.form-inline .btn-group {
- display: inline-block;
-}
-.form-search .input-append,
-.form-inline .input-append,
-.form-search .input-prepend,
-.form-inline .input-prepend {
- margin-bottom: 0;
-}
-.form-search .radio,
-.form-search .checkbox,
-.form-inline .radio,
-.form-inline .checkbox {
- padding-left: 0;
- margin-bottom: 0;
- vertical-align: middle;
-}
-.form-search .radio input[type="radio"],
-.form-search .checkbox input[type="checkbox"],
-.form-inline .radio input[type="radio"],
-.form-inline .checkbox input[type="checkbox"] {
- float: left;
- margin-right: 3px;
- margin-left: 0;
-}
-.control-group {
- margin-bottom: 10px;
-}
-legend + .control-group {
- margin-top: 20px;
- -webkit-margin-top-collapse: separate;
-}
-.form-horizontal .control-group {
- margin-bottom: 20px;
- *zoom: 1;
-}
-.form-horizontal .control-group:before,
-.form-horizontal .control-group:after {
- display: table;
- content: "";
- line-height: 0;
-}
-.form-horizontal .control-group:after {
- clear: both;
-}
-.form-horizontal .control-label {
- float: left;
- width: 160px;
- padding-top: 5px;
- text-align: right;
-}
-.form-horizontal .controls {
- *display: inline-block;
- *padding-left: 20px;
- margin-left: 180px;
- *margin-left: 0;
-}
-.form-horizontal .controls:first-child {
- *padding-left: 180px;
-}
-.form-horizontal .help-block {
- margin-bottom: 0;
-}
-.form-horizontal input + .help-block,
-.form-horizontal select + .help-block,
-.form-horizontal textarea + .help-block,
-.form-horizontal .uneditable-input + .help-block,
-.form-horizontal .input-prepend + .help-block,
-.form-horizontal .input-append + .help-block {
- margin-top: 10px;
-}
-.form-horizontal .form-actions {
- padding-left: 180px;
-}
-table {
- max-width: 100%;
- background-color: transparent;
- border-collapse: collapse;
- border-spacing: 0;
-}
-.table {
- width: 100%;
- margin-bottom: 20px;
-}
-.table th,
-.table td {
- padding: 8px;
- line-height: 20px;
- text-align: left;
- vertical-align: top;
- border-top: 1px solid #dddddd;
-}
-.table th {
- font-weight: bold;
-}
-.table thead th {
- vertical-align: bottom;
-}
-.table caption + thead tr:first-child th,
-.table caption + thead tr:first-child td,
-.table colgroup + thead tr:first-child th,
-.table colgroup + thead tr:first-child td,
-.table thead:first-child tr:first-child th,
-.table thead:first-child tr:first-child td {
- border-top: 0;
-}
-.table tbody + tbody {
- border-top: 2px solid #dddddd;
-}
-.table .table {
- background-color: #ffffff;
-}
-.table-condensed th,
-.table-condensed td {
- padding: 4px 5px;
-}
-.table-bordered {
- border: 1px solid #dddddd;
- border-collapse: separate;
- *border-collapse: collapse;
- border-left: 0;
- -webkit-border-radius: 4px;
- -moz-border-radius: 4px;
- border-radius: 4px;
-}
-.table-bordered th,
-.table-bordered td {
- border-left: 1px solid #dddddd;
-}
-.table-bordered caption + thead tr:first-child th,
-.table-bordered caption + tbody tr:first-child th,
-.table-bordered caption + tbody tr:first-child td,
-.table-bordered colgroup + thead tr:first-child th,
-.table-bordered colgroup + tbody tr:first-child th,
-.table-bordered colgroup + tbody tr:first-child td,
-.table-bordered thead:first-child tr:first-child th,
-.table-bordered tbody:first-child tr:first-child th,
-.table-bordered tbody:first-child tr:first-child td {
- border-top: 0;
-}
-.table-bordered thead:first-child tr:first-child > th:first-child,
-.table-bordered tbody:first-child tr:first-child > td:first-child,
-.table-bordered tbody:first-child tr:first-child > th:first-child {
- -webkit-border-top-left-radius: 4px;
- -moz-border-radius-topleft: 4px;
- border-top-left-radius: 4px;
-}
-.table-bordered thead:first-child tr:first-child > th:last-child,
-.table-bordered tbody:first-child tr:first-child > td:last-child,
-.table-bordered tbody:first-child tr:first-child > th:last-child {
- -webkit-border-top-right-radius: 4px;
- -moz-border-radius-topright: 4px;
- border-top-right-radius: 4px;
-}
-.table-bordered thead:last-child tr:last-child > th:first-child,
-.table-bordered tbody:last-child tr:last-child > td:first-child,
-.table-bordered tbody:last-child tr:last-child > th:first-child,
-.table-bordered tfoot:last-child tr:last-child > td:first-child,
-.table-bordered tfoot:last-child tr:last-child > th:first-child {
- -webkit-border-bottom-left-radius: 4px;
- -moz-border-radius-bottomleft: 4px;
- border-bottom-left-radius: 4px;
-}
-.table-bordered thead:last-child tr:last-child > th:last-child,
-.table-bordered tbody:last-child tr:last-child > td:last-child,
-.table-bordered tbody:last-child tr:last-child > th:last-child,
-.table-bordered tfoot:last-child tr:last-child > td:last-child,
-.table-bordered tfoot:last-child tr:last-child > th:last-child {
- -webkit-border-bottom-right-radius: 4px;
- -moz-border-radius-bottomright: 4px;
- border-bottom-right-radius: 4px;
-}
-.table-bordered tfoot + tbody:last-child tr:last-child td:first-child {
- -webkit-border-bottom-left-radius: 0;
- -moz-border-radius-bottomleft: 0;
- border-bottom-left-radius: 0;
-}
-.table-bordered tfoot + tbody:last-child tr:last-child td:last-child {
- -webkit-border-bottom-right-radius: 0;
- -moz-border-radius-bottomright: 0;
- border-bottom-right-radius: 0;
-}
-.table-bordered caption + thead tr:first-child th:first-child,
-.table-bordered caption + tbody tr:first-child td:first-child,
-.table-bordered colgroup + thead tr:first-child th:first-child,
-.table-bordered colgroup + tbody tr:first-child td:first-child {
- -webkit-border-top-left-radius: 4px;
- -moz-border-radius-topleft: 4px;
- border-top-left-radius: 4px;
-}
-.table-bordered caption + thead tr:first-child th:last-child,
-.table-bordered caption + tbody tr:first-child td:last-child,
-.table-bordered colgroup + thead tr:first-child th:last-child,
-.table-bordered colgroup + tbody tr:first-child td:last-child {
- -webkit-border-top-right-radius: 4px;
- -moz-border-radius-topright: 4px;
- border-top-right-radius: 4px;
-}
-.table-striped tbody > tr:nth-child(odd) > td,
-.table-striped tbody > tr:nth-child(odd) > th {
- background-color: #f9f9f9;
-}
-.table-hover tbody tr:hover > td,
-.table-hover tbody tr:hover > th {
- background-color: #f5f5f5;
-}
-table td[class*="span"],
-table th[class*="span"],
-.row-fluid table td[class*="span"],
-.row-fluid table th[class*="span"] {
- display: table-cell;
- float: none;
- margin-left: 0;
-}
-.table td.span1,
-.table th.span1 {
- float: none;
- width: 44px;
- margin-left: 0;
-}
-.table td.span2,
-.table th.span2 {
- float: none;
- width: 124px;
- margin-left: 0;
-}
-.table td.span3,
-.table th.span3 {
- float: none;
- width: 204px;
- margin-left: 0;
-}
-.table td.span4,
-.table th.span4 {
- float: none;
- width: 284px;
- margin-left: 0;
-}
-.table td.span5,
-.table th.span5 {
- float: none;
- width: 364px;
- margin-left: 0;
-}
-.table td.span6,
-.table th.span6 {
- float: none;
- width: 444px;
- margin-left: 0;
-}
-.table td.span7,
-.table th.span7 {
- float: none;
- width: 524px;
- margin-left: 0;
-}
-.table td.span8,
-.table th.span8 {
- float: none;
- width: 604px;
- margin-left: 0;
-}
-.table td.span9,
-.table th.span9 {
- float: none;
- width: 684px;
- margin-left: 0;
-}
-.table td.span10,
-.table th.span10 {
- float: none;
- width: 764px;
- margin-left: 0;
-}
-.table td.span11,
-.table th.span11 {
- float: none;
- width: 844px;
- margin-left: 0;
-}
-.table td.span12,
-.table th.span12 {
- float: none;
- width: 924px;
- margin-left: 0;
-}
-.table tbody tr.success > td {
- background-color: #dff0d8;
-}
-.table tbody tr.error > td {
- background-color: #f2dede;
-}
-.table tbody tr.warning > td {
- background-color: #fcf8e3;
-}
-.table tbody tr.info > td {
- background-color: #d9edf7;
-}
-.table-hover tbody tr.success:hover > td {
- background-color: #d0e9c6;
-}
-.table-hover tbody tr.error:hover > td {
- background-color: #ebcccc;
-}
-.table-hover tbody tr.warning:hover > td {
- background-color: #faf2cc;
-}
-.table-hover tbody tr.info:hover > td {
- background-color: #c4e3f3;
-}
-/*@import "bootstrap/sprites.less";*/
-.dropup,
-.dropdown {
- position: relative;
-}
-.dropdown-toggle {
- *margin-bottom: -3px;
-}
-.dropdown-toggle:active,
-.open .dropdown-toggle {
- outline: 0;
-}
-.caret {
- display: inline-block;
- width: 0;
- height: 0;
- vertical-align: top;
- border-top: 4px solid #000000;
- border-right: 4px solid transparent;
- border-left: 4px solid transparent;
- content: "";
-}
-.dropdown .caret {
- margin-top: 8px;
- margin-left: 2px;
-}
-.dropdown-menu {
- position: absolute;
- top: 100%;
- left: 0;
- z-index: 1000;
- display: none;
- float: left;
- min-width: 160px;
- padding: 5px 0;
- margin: 2px 0 0;
- list-style: none;
- background-color: #ffffff;
- border: 1px solid #ccc;
- border: 1px solid rgba(0, 0, 0, 0.2);
- *border-right-width: 2px;
- *border-bottom-width: 2px;
- -webkit-border-radius: 6px;
- -moz-border-radius: 6px;
- border-radius: 6px;
- -webkit-box-shadow: 0 5px 10px rgba(0, 0, 0, 0.2);
- -moz-box-shadow: 0 5px 10px rgba(0, 0, 0, 0.2);
- box-shadow: 0 5px 10px rgba(0, 0, 0, 0.2);
- -webkit-background-clip: padding-box;
- -moz-background-clip: padding;
- background-clip: padding-box;
-}
-.dropdown-menu.pull-right {
- right: 0;
- left: auto;
-}
-.dropdown-menu .divider {
- *width: 100%;
- height: 1px;
- margin: 9px 1px;
- *margin: -5px 0 5px;
- overflow: hidden;
- background-color: #e5e5e5;
- border-bottom: 1px solid #ffffff;
-}
-.dropdown-menu > li > a {
- display: block;
- padding: 3px 20px;
- clear: both;
- font-weight: normal;
- line-height: 20px;
- color: #333333;
- white-space: nowrap;
-}
-.dropdown-menu > li > a:hover,
-.dropdown-menu > li > a:focus,
-.dropdown-submenu:hover > a,
-.dropdown-submenu:focus > a {
- text-decoration: none;
- color: #ffffff;
- background-color: #0081c2;
- background-image: -moz-linear-gradient(top, #0088cc, #0077b3);
- background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#0088cc), to(#0077b3));
- background-image: -webkit-linear-gradient(top, #0088cc, #0077b3);
- background-image: -o-linear-gradient(top, #0088cc, #0077b3);
- background-image: linear-gradient(to bottom, #0088cc, #0077b3);
- background-repeat: repeat-x;
- filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff0088cc', endColorstr='#ff0077b3', GradientType=0);
-}
-.dropdown-menu > .active > a,
-.dropdown-menu > .active > a:hover,
-.dropdown-menu > .active > a:focus {
- color: #ffffff;
- text-decoration: none;
- outline: 0;
- background-color: #0081c2;
- background-image: -moz-linear-gradient(top, #0088cc, #0077b3);
- background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#0088cc), to(#0077b3));
- background-image: -webkit-linear-gradient(top, #0088cc, #0077b3);
- background-image: -o-linear-gradient(top, #0088cc, #0077b3);
- background-image: linear-gradient(to bottom, #0088cc, #0077b3);
- background-repeat: repeat-x;
- filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff0088cc', endColorstr='#ff0077b3', GradientType=0);
-}
-.dropdown-menu > .disabled > a,
-.dropdown-menu > .disabled > a:hover,
-.dropdown-menu > .disabled > a:focus {
- color: #999999;
-}
-.dropdown-menu > .disabled > a:hover,
-.dropdown-menu > .disabled > a:focus {
- text-decoration: none;
- background-color: transparent;
- background-image: none;
- filter: progid:DXImageTransform.Microsoft.gradient(enabled = false);
- cursor: default;
-}
-.open {
- *z-index: 1000;
-}
-.open > .dropdown-menu {
- display: block;
-}
-.pull-right > .dropdown-menu {
- right: 0;
- left: auto;
-}
-.dropup .caret,
-.navbar-fixed-bottom .dropdown .caret {
- border-top: 0;
- border-bottom: 4px solid #000000;
- content: "";
-}
-.dropup .dropdown-menu,
-.navbar-fixed-bottom .dropdown .dropdown-menu {
- top: auto;
- bottom: 100%;
- margin-bottom: 1px;
-}
-.dropdown-submenu {
- position: relative;
-}
-.dropdown-submenu > .dropdown-menu {
- top: 0;
- left: 100%;
- margin-top: -6px;
- margin-left: -1px;
- -webkit-border-radius: 0 6px 6px 6px;
- -moz-border-radius: 0 6px 6px 6px;
- border-radius: 0 6px 6px 6px;
-}
-.dropdown-submenu:hover > .dropdown-menu {
- display: block;
-}
-.dropup .dropdown-submenu > .dropdown-menu {
- top: auto;
- bottom: 0;
- margin-top: 0;
- margin-bottom: -2px;
- -webkit-border-radius: 5px 5px 5px 0;
- -moz-border-radius: 5px 5px 5px 0;
- border-radius: 5px 5px 5px 0;
-}
-.dropdown-submenu > a:after {
- display: block;
- content: " ";
- float: right;
- width: 0;
- height: 0;
- border-color: transparent;
- border-style: solid;
- border-width: 5px 0 5px 5px;
- border-left-color: #cccccc;
- margin-top: 5px;
- margin-right: -10px;
-}
-.dropdown-submenu:hover > a:after {
- border-left-color: #ffffff;
-}
-.dropdown-submenu.pull-left {
- float: none;
-}
-.dropdown-submenu.pull-left > .dropdown-menu {
- left: -100%;
- margin-left: 10px;
- -webkit-border-radius: 6px 0 6px 6px;
- -moz-border-radius: 6px 0 6px 6px;
- border-radius: 6px 0 6px 6px;
-}
-.dropdown .dropdown-menu .nav-header {
- padding-left: 20px;
- padding-right: 20px;
-}
-.typeahead {
- z-index: 1051;
- margin-top: 2px;
- -webkit-border-radius: 4px;
- -moz-border-radius: 4px;
- border-radius: 4px;
-}
-.well {
- min-height: 20px;
- padding: 19px;
- margin-bottom: 20px;
- background-color: #f5f5f5;
- border: 1px solid #e3e3e3;
- -webkit-border-radius: 4px;
- -moz-border-radius: 4px;
- border-radius: 4px;
- -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.05);
- -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.05);
- box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.05);
-}
-.well blockquote {
- border-color: #ddd;
- border-color: rgba(0, 0, 0, 0.15);
-}
-.well-large {
- padding: 24px;
- -webkit-border-radius: 6px;
- -moz-border-radius: 6px;
- border-radius: 6px;
-}
-.well-small {
- padding: 9px;
- -webkit-border-radius: 3px;
- -moz-border-radius: 3px;
- border-radius: 3px;
-}
-/*@import "bootstrap/component-animations.less";*/
-/*@import "bootstrap/close.less";*/
-.btn {
- display: inline-block;
- *display: inline;
- /* IE7 inline-block hack */
-
- *zoom: 1;
- padding: 4px 12px;
- margin-bottom: 0;
- font-size: 13px;
- line-height: 20px;
- text-align: center;
- vertical-align: middle;
- cursor: pointer;
- color: #333333;
- text-shadow: 0 1px 1px rgba(255, 255, 255, 0.75);
- background-color: #f5f5f5;
- background-image: -moz-linear-gradient(top, #ffffff, #e6e6e6);
- background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#ffffff), to(#e6e6e6));
- background-image: -webkit-linear-gradient(top, #ffffff, #e6e6e6);
- background-image: -o-linear-gradient(top, #ffffff, #e6e6e6);
- background-image: linear-gradient(to bottom, #ffffff, #e6e6e6);
- background-repeat: repeat-x;
- filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffffffff', endColorstr='#ffe6e6e6', GradientType=0);
- border-color: #e6e6e6 #e6e6e6 #bfbfbf;
- border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
- *background-color: #e6e6e6;
- /* Darken IE7 buttons by default so they stand out more given they won't have borders */
-
- filter: progid:DXImageTransform.Microsoft.gradient(enabled = false);
- border: 1px solid #cccccc;
- *border: 0;
- border-bottom-color: #b3b3b3;
- -webkit-border-radius: 4px;
- -moz-border-radius: 4px;
- border-radius: 4px;
- *margin-left: .3em;
- -webkit-box-shadow: inset 0 1px 0 rgba(255,255,255,.2), 0 1px 2px rgba(0,0,0,.05);
- -moz-box-shadow: inset 0 1px 0 rgba(255,255,255,.2), 0 1px 2px rgba(0,0,0,.05);
- box-shadow: inset 0 1px 0 rgba(255,255,255,.2), 0 1px 2px rgba(0,0,0,.05);
-}
-.btn:hover,
-.btn:focus,
-.btn:active,
-.btn.active,
-.btn.disabled,
-.btn[disabled] {
- color: #333333;
- background-color: #e6e6e6;
- *background-color: #d9d9d9;
-}
-.btn:active,
-.btn.active {
- background-color: #cccccc \9;
-}
-.btn:first-child {
- *margin-left: 0;
-}
-.btn:hover,
-.btn:focus {
- color: #333333;
- text-decoration: none;
- background-position: 0 -15px;
- -webkit-transition: background-position 0.1s linear;
- -moz-transition: background-position 0.1s linear;
- -o-transition: background-position 0.1s linear;
- transition: background-position 0.1s linear;
-}
-.btn:focus {
- outline: thin dotted #333;
- outline: 5px auto -webkit-focus-ring-color;
- outline-offset: -2px;
-}
-.btn.active,
-.btn:active {
- background-image: none;
- outline: 0;
- -webkit-box-shadow: inset 0 2px 4px rgba(0,0,0,.15), 0 1px 2px rgba(0,0,0,.05);
- -moz-box-shadow: inset 0 2px 4px rgba(0,0,0,.15), 0 1px 2px rgba(0,0,0,.05);
- box-shadow: inset 0 2px 4px rgba(0,0,0,.15), 0 1px 2px rgba(0,0,0,.05);
-}
-.btn.disabled,
-.btn[disabled] {
- cursor: default;
- background-image: none;
- opacity: 0.65;
- filter: alpha(opacity=65);
- -webkit-box-shadow: none;
- -moz-box-shadow: none;
- box-shadow: none;
-}
-.btn-large {
- padding: 11px 19px;
- font-size: 16.25px;
- -webkit-border-radius: 6px;
- -moz-border-radius: 6px;
- border-radius: 6px;
-}
-.btn-large [class^="icon-"],
-.btn-large [class*=" icon-"] {
- margin-top: 4px;
-}
-.btn-small {
- padding: 2px 10px;
- font-size: 11.049999999999999px;
- -webkit-border-radius: 3px;
- -moz-border-radius: 3px;
- border-radius: 3px;
-}
-.btn-small [class^="icon-"],
-.btn-small [class*=" icon-"] {
- margin-top: 0;
-}
-.btn-mini [class^="icon-"],
-.btn-mini [class*=" icon-"] {
- margin-top: -1px;
-}
-.btn-mini {
- padding: 0 6px;
- font-size: 9.75px;
- -webkit-border-radius: 3px;
- -moz-border-radius: 3px;
- border-radius: 3px;
-}
-.btn-block {
- display: block;
- width: 100%;
- padding-left: 0;
- padding-right: 0;
- -webkit-box-sizing: border-box;
- -moz-box-sizing: border-box;
- box-sizing: border-box;
-}
-.btn-block + .btn-block {
- margin-top: 5px;
-}
-input[type="submit"].btn-block,
-input[type="reset"].btn-block,
-input[type="button"].btn-block {
- width: 100%;
-}
-.btn-primary.active,
-.btn-warning.active,
-.btn-danger.active,
-.btn-success.active,
-.btn-info.active,
-.btn-inverse.active {
- color: rgba(255, 255, 255, 0.75);
-}
-.btn-primary {
- color: #ffffff;
- text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25);
- background-color: #006dcc;
- background-image: -moz-linear-gradient(top, #0088cc, #0044cc);
- background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#0088cc), to(#0044cc));
- background-image: -webkit-linear-gradient(top, #0088cc, #0044cc);
- background-image: -o-linear-gradient(top, #0088cc, #0044cc);
- background-image: linear-gradient(to bottom, #0088cc, #0044cc);
- background-repeat: repeat-x;
- filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff0088cc', endColorstr='#ff0044cc', GradientType=0);
- border-color: #0044cc #0044cc #002a80;
- border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
- *background-color: #0044cc;
- /* Darken IE7 buttons by default so they stand out more given they won't have borders */
-
- filter: progid:DXImageTransform.Microsoft.gradient(enabled = false);
-}
-.btn-primary:hover,
-.btn-primary:focus,
-.btn-primary:active,
-.btn-primary.active,
-.btn-primary.disabled,
-.btn-primary[disabled] {
- color: #ffffff;
- background-color: #0044cc;
- *background-color: #003bb3;
-}
-.btn-primary:active,
-.btn-primary.active {
- background-color: #003399 \9;
-}
-.btn-warning {
- color: #ffffff;
- text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25);
- background-color: #faa732;
- background-image: -moz-linear-gradient(top, #fbb450, #f89406);
- background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#fbb450), to(#f89406));
- background-image: -webkit-linear-gradient(top, #fbb450, #f89406);
- background-image: -o-linear-gradient(top, #fbb450, #f89406);
- background-image: linear-gradient(to bottom, #fbb450, #f89406);
- background-repeat: repeat-x;
- filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#fffbb450', endColorstr='#fff89406', GradientType=0);
- border-color: #f89406 #f89406 #ad6704;
- border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
- *background-color: #f89406;
- /* Darken IE7 buttons by default so they stand out more given they won't have borders */
-
- filter: progid:DXImageTransform.Microsoft.gradient(enabled = false);
-}
-.btn-warning:hover,
-.btn-warning:focus,
-.btn-warning:active,
-.btn-warning.active,
-.btn-warning.disabled,
-.btn-warning[disabled] {
- color: #ffffff;
- background-color: #f89406;
- *background-color: #df8505;
-}
-.btn-warning:active,
-.btn-warning.active {
- background-color: #c67605 \9;
-}
-.btn-danger {
- color: #ffffff;
- text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25);
- background-color: #da4f49;
- background-image: -moz-linear-gradient(top, #ee5f5b, #bd362f);
- background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#ee5f5b), to(#bd362f));
- background-image: -webkit-linear-gradient(top, #ee5f5b, #bd362f);
- background-image: -o-linear-gradient(top, #ee5f5b, #bd362f);
- background-image: linear-gradient(to bottom, #ee5f5b, #bd362f);
- background-repeat: repeat-x;
- filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffee5f5b', endColorstr='#ffbd362f', GradientType=0);
- border-color: #bd362f #bd362f #802420;
- border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
- *background-color: #bd362f;
- /* Darken IE7 buttons by default so they stand out more given they won't have borders */
-
- filter: progid:DXImageTransform.Microsoft.gradient(enabled = false);
-}
-.btn-danger:hover,
-.btn-danger:focus,
-.btn-danger:active,
-.btn-danger.active,
-.btn-danger.disabled,
-.btn-danger[disabled] {
- color: #ffffff;
- background-color: #bd362f;
- *background-color: #a9302a;
-}
-.btn-danger:active,
-.btn-danger.active {
- background-color: #942a25 \9;
-}
-.btn-success {
- color: #ffffff;
- text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25);
- background-color: #5bb75b;
- background-image: -moz-linear-gradient(top, #62c462, #51a351);
- background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#62c462), to(#51a351));
- background-image: -webkit-linear-gradient(top, #62c462, #51a351);
- background-image: -o-linear-gradient(top, #62c462, #51a351);
- background-image: linear-gradient(to bottom, #62c462, #51a351);
- background-repeat: repeat-x;
- filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff62c462', endColorstr='#ff51a351', GradientType=0);
- border-color: #51a351 #51a351 #387038;
- border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
- *background-color: #51a351;
- /* Darken IE7 buttons by default so they stand out more given they won't have borders */
-
- filter: progid:DXImageTransform.Microsoft.gradient(enabled = false);
-}
-.btn-success:hover,
-.btn-success:focus,
-.btn-success:active,
-.btn-success.active,
-.btn-success.disabled,
-.btn-success[disabled] {
- color: #ffffff;
- background-color: #51a351;
- *background-color: #499249;
-}
-.btn-success:active,
-.btn-success.active {
- background-color: #408140 \9;
-}
-.btn-info {
- color: #ffffff;
- text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25);
- background-color: #49afcd;
- background-image: -moz-linear-gradient(top, #5bc0de, #2f96b4);
- background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#5bc0de), to(#2f96b4));
- background-image: -webkit-linear-gradient(top, #5bc0de, #2f96b4);
- background-image: -o-linear-gradient(top, #5bc0de, #2f96b4);
- background-image: linear-gradient(to bottom, #5bc0de, #2f96b4);
- background-repeat: repeat-x;
- filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff5bc0de', endColorstr='#ff2f96b4', GradientType=0);
- border-color: #2f96b4 #2f96b4 #1f6377;
- border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
- *background-color: #2f96b4;
- /* Darken IE7 buttons by default so they stand out more given they won't have borders */
-
- filter: progid:DXImageTransform.Microsoft.gradient(enabled = false);
-}
-.btn-info:hover,
-.btn-info:focus,
-.btn-info:active,
-.btn-info.active,
-.btn-info.disabled,
-.btn-info[disabled] {
- color: #ffffff;
- background-color: #2f96b4;
- *background-color: #2a85a0;
-}
-.btn-info:active,
-.btn-info.active {
- background-color: #24748c \9;
-}
-.btn-inverse {
- color: #ffffff;
- text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25);
- background-color: #363636;
- background-image: -moz-linear-gradient(top, #444444, #222222);
- background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#444444), to(#222222));
- background-image: -webkit-linear-gradient(top, #444444, #222222);
- background-image: -o-linear-gradient(top, #444444, #222222);
- background-image: linear-gradient(to bottom, #444444, #222222);
- background-repeat: repeat-x;
- filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff444444', endColorstr='#ff222222', GradientType=0);
- border-color: #222222 #222222 #000000;
- border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
- *background-color: #222222;
- /* Darken IE7 buttons by default so they stand out more given they won't have borders */
-
- filter: progid:DXImageTransform.Microsoft.gradient(enabled = false);
-}
-.btn-inverse:hover,
-.btn-inverse:focus,
-.btn-inverse:active,
-.btn-inverse.active,
-.btn-inverse.disabled,
-.btn-inverse[disabled] {
- color: #ffffff;
- background-color: #222222;
- *background-color: #151515;
-}
-.btn-inverse:active,
-.btn-inverse.active {
- background-color: #080808 \9;
-}
-button.btn,
-input[type="submit"].btn {
- *padding-top: 3px;
- *padding-bottom: 3px;
-}
-button.btn::-moz-focus-inner,
-input[type="submit"].btn::-moz-focus-inner {
- padding: 0;
- border: 0;
-}
-button.btn.btn-large,
-input[type="submit"].btn.btn-large {
- *padding-top: 7px;
- *padding-bottom: 7px;
-}
-button.btn.btn-small,
-input[type="submit"].btn.btn-small {
- *padding-top: 3px;
- *padding-bottom: 3px;
-}
-button.btn.btn-mini,
-input[type="submit"].btn.btn-mini {
- *padding-top: 1px;
- *padding-bottom: 1px;
-}
-.btn-link,
-.btn-link:active,
-.btn-link[disabled] {
- background-color: transparent;
- background-image: none;
- -webkit-box-shadow: none;
- -moz-box-shadow: none;
- box-shadow: none;
-}
-.btn-link {
- border-color: transparent;
- cursor: pointer;
- color: #0088cc;
- -webkit-border-radius: 0;
- -moz-border-radius: 0;
- border-radius: 0;
-}
-.btn-link:hover,
-.btn-link:focus {
- color: #005580;
- text-decoration: underline;
- background-color: transparent;
-}
-.btn-link[disabled]:hover,
-.btn-link[disabled]:focus {
- color: #333333;
- text-decoration: none;
-}
-.btn-group {
- position: relative;
- display: inline-block;
- *display: inline;
- /* IE7 inline-block hack */
-
- *zoom: 1;
- font-size: 0;
- vertical-align: middle;
- white-space: nowrap;
- *margin-left: .3em;
-}
-.btn-group:first-child {
- *margin-left: 0;
-}
-.btn-group + .btn-group {
- margin-left: 5px;
-}
-.btn-toolbar {
- font-size: 0;
- margin-top: 10px;
- margin-bottom: 10px;
-}
-.btn-toolbar > .btn + .btn,
-.btn-toolbar > .btn-group + .btn,
-.btn-toolbar > .btn + .btn-group {
- margin-left: 5px;
-}
-.btn-group > .btn {
- position: relative;
- -webkit-border-radius: 0;
- -moz-border-radius: 0;
- border-radius: 0;
-}
-.btn-group > .btn + .btn {
- margin-left: -1px;
-}
-.btn-group > .btn,
-.btn-group > .dropdown-menu,
-.btn-group > .popover {
- font-size: 13px;
-}
-.btn-group > .btn-mini {
- font-size: 9.75px;
-}
-.btn-group > .btn-small {
- font-size: 11.049999999999999px;
-}
-.btn-group > .btn-large {
- font-size: 16.25px;
-}
-.btn-group > .btn:first-child {
- margin-left: 0;
- -webkit-border-top-left-radius: 4px;
- -moz-border-radius-topleft: 4px;
- border-top-left-radius: 4px;
- -webkit-border-bottom-left-radius: 4px;
- -moz-border-radius-bottomleft: 4px;
- border-bottom-left-radius: 4px;
-}
-.btn-group > .btn:last-child,
-.btn-group > .dropdown-toggle {
- -webkit-border-top-right-radius: 4px;
- -moz-border-radius-topright: 4px;
- border-top-right-radius: 4px;
- -webkit-border-bottom-right-radius: 4px;
- -moz-border-radius-bottomright: 4px;
- border-bottom-right-radius: 4px;
-}
-.btn-group > .btn.large:first-child {
- margin-left: 0;
- -webkit-border-top-left-radius: 6px;
- -moz-border-radius-topleft: 6px;
- border-top-left-radius: 6px;
- -webkit-border-bottom-left-radius: 6px;
- -moz-border-radius-bottomleft: 6px;
- border-bottom-left-radius: 6px;
-}
-.btn-group > .btn.large:last-child,
-.btn-group > .large.dropdown-toggle {
- -webkit-border-top-right-radius: 6px;
- -moz-border-radius-topright: 6px;
- border-top-right-radius: 6px;
- -webkit-border-bottom-right-radius: 6px;
- -moz-border-radius-bottomright: 6px;
- border-bottom-right-radius: 6px;
-}
-.btn-group > .btn:hover,
-.btn-group > .btn:focus,
-.btn-group > .btn:active,
-.btn-group > .btn.active {
- z-index: 2;
-}
-.btn-group .dropdown-toggle:active,
-.btn-group.open .dropdown-toggle {
- outline: 0;
-}
-.btn-group > .btn + .dropdown-toggle {
- padding-left: 8px;
- padding-right: 8px;
- -webkit-box-shadow: inset 1px 0 0 rgba(255,255,255,.125), inset 0 1px 0 rgba(255,255,255,.2), 0 1px 2px rgba(0,0,0,.05);
- -moz-box-shadow: inset 1px 0 0 rgba(255,255,255,.125), inset 0 1px 0 rgba(255,255,255,.2), 0 1px 2px rgba(0,0,0,.05);
- box-shadow: inset 1px 0 0 rgba(255,255,255,.125), inset 0 1px 0 rgba(255,255,255,.2), 0 1px 2px rgba(0,0,0,.05);
- *padding-top: 5px;
- *padding-bottom: 5px;
-}
-.btn-group > .btn-mini + .dropdown-toggle {
- padding-left: 5px;
- padding-right: 5px;
- *padding-top: 2px;
- *padding-bottom: 2px;
-}
-.btn-group > .btn-small + .dropdown-toggle {
- *padding-top: 5px;
- *padding-bottom: 4px;
-}
-.btn-group > .btn-large + .dropdown-toggle {
- padding-left: 12px;
- padding-right: 12px;
- *padding-top: 7px;
- *padding-bottom: 7px;
-}
-.btn-group.open .dropdown-toggle {
- background-image: none;
- -webkit-box-shadow: inset 0 2px 4px rgba(0,0,0,.15), 0 1px 2px rgba(0,0,0,.05);
- -moz-box-shadow: inset 0 2px 4px rgba(0,0,0,.15), 0 1px 2px rgba(0,0,0,.05);
- box-shadow: inset 0 2px 4px rgba(0,0,0,.15), 0 1px 2px rgba(0,0,0,.05);
-}
-.btn-group.open .btn.dropdown-toggle {
- background-color: #e6e6e6;
-}
-.btn-group.open .btn-primary.dropdown-toggle {
- background-color: #0044cc;
-}
-.btn-group.open .btn-warning.dropdown-toggle {
- background-color: #f89406;
-}
-.btn-group.open .btn-danger.dropdown-toggle {
- background-color: #bd362f;
-}
-.btn-group.open .btn-success.dropdown-toggle {
- background-color: #51a351;
-}
-.btn-group.open .btn-info.dropdown-toggle {
- background-color: #2f96b4;
-}
-.btn-group.open .btn-inverse.dropdown-toggle {
- background-color: #222222;
-}
-.btn .caret {
- margin-top: 8px;
- margin-left: 0;
-}
-.btn-large .caret {
- margin-top: 6px;
-}
-.btn-large .caret {
- border-left-width: 5px;
- border-right-width: 5px;
- border-top-width: 5px;
-}
-.btn-mini .caret,
-.btn-small .caret {
- margin-top: 8px;
-}
-.dropup .btn-large .caret {
- border-bottom-width: 5px;
-}
-.btn-primary .caret,
-.btn-warning .caret,
-.btn-danger .caret,
-.btn-info .caret,
-.btn-success .caret,
-.btn-inverse .caret {
- border-top-color: #ffffff;
- border-bottom-color: #ffffff;
-}
-.btn-group-vertical {
- display: inline-block;
- *display: inline;
- /* IE7 inline-block hack */
-
- *zoom: 1;
-}
-.btn-group-vertical > .btn {
- display: block;
- float: none;
- max-width: 100%;
- -webkit-border-radius: 0;
- -moz-border-radius: 0;
- border-radius: 0;
-}
-.btn-group-vertical > .btn + .btn {
- margin-left: 0;
- margin-top: -1px;
-}
-.btn-group-vertical > .btn:first-child {
- -webkit-border-radius: 4px 4px 0 0;
- -moz-border-radius: 4px 4px 0 0;
- border-radius: 4px 4px 0 0;
-}
-.btn-group-vertical > .btn:last-child {
- -webkit-border-radius: 0 0 4px 4px;
- -moz-border-radius: 0 0 4px 4px;
- border-radius: 0 0 4px 4px;
-}
-.btn-group-vertical > .btn-large:first-child {
- -webkit-border-radius: 6px 6px 0 0;
- -moz-border-radius: 6px 6px 0 0;
- border-radius: 6px 6px 0 0;
-}
-.btn-group-vertical > .btn-large:last-child {
- -webkit-border-radius: 0 0 6px 6px;
- -moz-border-radius: 0 0 6px 6px;
- border-radius: 0 0 6px 6px;
-}
-.alert {
- padding: 8px 35px 8px 14px;
- margin-bottom: 20px;
- text-shadow: 0 1px 0 rgba(255, 255, 255, 0.5);
- background-color: #fcf8e3;
- border: 1px solid #fbeed5;
- -webkit-border-radius: 4px;
- -moz-border-radius: 4px;
- border-radius: 4px;
-}
-.alert,
-.alert h4 {
- color: #c09853;
-}
-.alert h4 {
- margin: 0;
-}
-.alert .close {
- position: relative;
- top: -2px;
- right: -21px;
- line-height: 20px;
-}
-.alert-success {
- background-color: #dff0d8;
- border-color: #d6e9c6;
- color: #468847;
-}
-.alert-success h4 {
- color: #468847;
-}
-.alert-danger,
-.alert-error {
- background-color: #f2dede;
- border-color: #eed3d7;
- color: #b94a48;
-}
-.alert-danger h4,
-.alert-error h4 {
- color: #b94a48;
-}
-.alert-info {
- background-color: #d9edf7;
- border-color: #bce8f1;
- color: #3a87ad;
-}
-.alert-info h4 {
- color: #3a87ad;
-}
-.alert-block {
- padding-top: 14px;
- padding-bottom: 14px;
-}
-.alert-block > p,
-.alert-block > ul {
- margin-bottom: 0;
-}
-.alert-block p + p {
- margin-top: 5px;
-}
-.nav {
- margin-left: 0;
- margin-bottom: 20px;
- list-style: none;
-}
-.nav > li > a {
- display: block;
-}
-.nav > li > a:hover,
-.nav > li > a:focus {
- text-decoration: none;
- background-color: #eeeeee;
-}
-.nav > li > a > img {
- max-width: none;
-}
-.nav > .pull-right {
- float: right;
-}
-.nav-header {
- display: block;
- padding: 3px 15px;
- font-size: 11px;
- font-weight: bold;
- line-height: 20px;
- color: #999999;
- text-shadow: 0 1px 0 rgba(255, 255, 255, 0.5);
- text-transform: uppercase;
-}
-.nav li + .nav-header {
- margin-top: 9px;
-}
-.nav-list {
- padding-left: 15px;
- padding-right: 15px;
- margin-bottom: 0;
-}
-.nav-list > li > a,
-.nav-list .nav-header {
- margin-left: -15px;
- margin-right: -15px;
- text-shadow: 0 1px 0 rgba(255, 255, 255, 0.5);
-}
-.nav-list > li > a {
- padding: 3px 15px;
-}
-.nav-list > .active > a,
-.nav-list > .active > a:hover,
-.nav-list > .active > a:focus {
- color: #ffffff;
- text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.2);
- background-color: #0088cc;
-}
-.nav-list [class^="icon-"],
-.nav-list [class*=" icon-"] {
- margin-right: 2px;
-}
-.nav-list .divider {
- *width: 100%;
- height: 1px;
- margin: 9px 1px;
- *margin: -5px 0 5px;
- overflow: hidden;
- background-color: #e5e5e5;
- border-bottom: 1px solid #ffffff;
-}
-.nav-tabs,
-.nav-pills {
- *zoom: 1;
-}
-.nav-tabs:before,
-.nav-pills:before,
-.nav-tabs:after,
-.nav-pills:after {
- display: table;
- content: "";
- line-height: 0;
-}
-.nav-tabs:after,
-.nav-pills:after {
- clear: both;
-}
-.nav-tabs > li,
-.nav-pills > li {
- float: left;
-}
-.nav-tabs > li > a,
-.nav-pills > li > a {
- padding-right: 12px;
- padding-left: 12px;
- margin-right: 2px;
- line-height: 14px;
-}
-.nav-tabs {
- border-bottom: 1px solid #ddd;
-}
-.nav-tabs > li {
- margin-bottom: -1px;
-}
-.nav-tabs > li > a {
- padding-top: 8px;
- padding-bottom: 8px;
- line-height: 20px;
- border: 1px solid transparent;
- -webkit-border-radius: 4px 4px 0 0;
- -moz-border-radius: 4px 4px 0 0;
- border-radius: 4px 4px 0 0;
-}
-.nav-tabs > li > a:hover,
-.nav-tabs > li > a:focus {
- border-color: #eeeeee #eeeeee #dddddd;
-}
-.nav-tabs > .active > a,
-.nav-tabs > .active > a:hover,
-.nav-tabs > .active > a:focus {
- color: #555555;
- background-color: #ffffff;
- border: 1px solid #ddd;
- border-bottom-color: transparent;
- cursor: default;
-}
-.nav-pills > li > a {
- padding-top: 8px;
- padding-bottom: 8px;
- margin-top: 2px;
- margin-bottom: 2px;
- -webkit-border-radius: 5px;
- -moz-border-radius: 5px;
- border-radius: 5px;
-}
-.nav-pills > .active > a,
-.nav-pills > .active > a:hover,
-.nav-pills > .active > a:focus {
- color: #ffffff;
- background-color: #0088cc;
-}
-.nav-stacked > li {
- float: none;
-}
-.nav-stacked > li > a {
- margin-right: 0;
-}
-.nav-tabs.nav-stacked {
- border-bottom: 0;
-}
-.nav-tabs.nav-stacked > li > a {
- border: 1px solid #ddd;
- -webkit-border-radius: 0;
- -moz-border-radius: 0;
- border-radius: 0;
-}
-.nav-tabs.nav-stacked > li:first-child > a {
- -webkit-border-top-right-radius: 4px;
- -moz-border-radius-topright: 4px;
- border-top-right-radius: 4px;
- -webkit-border-top-left-radius: 4px;
- -moz-border-radius-topleft: 4px;
- border-top-left-radius: 4px;
-}
-.nav-tabs.nav-stacked > li:last-child > a {
- -webkit-border-bottom-right-radius: 4px;
- -moz-border-radius-bottomright: 4px;
- border-bottom-right-radius: 4px;
- -webkit-border-bottom-left-radius: 4px;
- -moz-border-radius-bottomleft: 4px;
- border-bottom-left-radius: 4px;
-}
-.nav-tabs.nav-stacked > li > a:hover,
-.nav-tabs.nav-stacked > li > a:focus {
- border-color: #ddd;
- z-index: 2;
-}
-.nav-pills.nav-stacked > li > a {
- margin-bottom: 3px;
-}
-.nav-pills.nav-stacked > li:last-child > a {
- margin-bottom: 1px;
-}
-.nav-tabs .dropdown-menu {
- -webkit-border-radius: 0 0 6px 6px;
- -moz-border-radius: 0 0 6px 6px;
- border-radius: 0 0 6px 6px;
-}
-.nav-pills .dropdown-menu {
- -webkit-border-radius: 6px;
- -moz-border-radius: 6px;
- border-radius: 6px;
-}
-.nav .dropdown-toggle .caret {
- border-top-color: #0088cc;
- border-bottom-color: #0088cc;
- margin-top: 6px;
-}
-.nav .dropdown-toggle:hover .caret,
-.nav .dropdown-toggle:focus .caret {
- border-top-color: #005580;
- border-bottom-color: #005580;
-}
-/* move down carets for tabs */
-.nav-tabs .dropdown-toggle .caret {
- margin-top: 8px;
-}
-.nav .active .dropdown-toggle .caret {
- border-top-color: #fff;
- border-bottom-color: #fff;
-}
-.nav-tabs .active .dropdown-toggle .caret {
- border-top-color: #555555;
- border-bottom-color: #555555;
-}
-.nav > .dropdown.active > a:hover,
-.nav > .dropdown.active > a:focus {
- cursor: pointer;
-}
-.nav-tabs .open .dropdown-toggle,
-.nav-pills .open .dropdown-toggle,
-.nav > li.dropdown.open.active > a:hover,
-.nav > li.dropdown.open.active > a:focus {
- color: #ffffff;
- background-color: #999999;
- border-color: #999999;
-}
-.nav li.dropdown.open .caret,
-.nav li.dropdown.open.active .caret,
-.nav li.dropdown.open a:hover .caret,
-.nav li.dropdown.open a:focus .caret {
- border-top-color: #ffffff;
- border-bottom-color: #ffffff;
- opacity: 1;
- filter: alpha(opacity=100);
-}
-.tabs-stacked .open > a:hover,
-.tabs-stacked .open > a:focus {
- border-color: #999999;
-}
-.tabbable {
- *zoom: 1;
-}
-.tabbable:before,
-.tabbable:after {
- display: table;
- content: "";
- line-height: 0;
-}
-.tabbable:after {
- clear: both;
-}
-.tab-content {
- overflow: auto;
-}
-.tabs-below > .nav-tabs,
-.tabs-right > .nav-tabs,
-.tabs-left > .nav-tabs {
- border-bottom: 0;
-}
-.tab-content > .tab-pane,
-.pill-content > .pill-pane {
- display: none;
-}
-.tab-content > .active,
-.pill-content > .active {
- display: block;
-}
-.tabs-below > .nav-tabs {
- border-top: 1px solid #ddd;
-}
-.tabs-below > .nav-tabs > li {
- margin-top: -1px;
- margin-bottom: 0;
-}
-.tabs-below > .nav-tabs > li > a {
- -webkit-border-radius: 0 0 4px 4px;
- -moz-border-radius: 0 0 4px 4px;
- border-radius: 0 0 4px 4px;
-}
-.tabs-below > .nav-tabs > li > a:hover,
-.tabs-below > .nav-tabs > li > a:focus {
- border-bottom-color: transparent;
- border-top-color: #ddd;
-}
-.tabs-below > .nav-tabs > .active > a,
-.tabs-below > .nav-tabs > .active > a:hover,
-.tabs-below > .nav-tabs > .active > a:focus {
- border-color: transparent #ddd #ddd #ddd;
-}
-.tabs-left > .nav-tabs > li,
-.tabs-right > .nav-tabs > li {
- float: none;
-}
-.tabs-left > .nav-tabs > li > a,
-.tabs-right > .nav-tabs > li > a {
- min-width: 74px;
- margin-right: 0;
- margin-bottom: 3px;
-}
-.tabs-left > .nav-tabs {
- float: left;
- margin-right: 19px;
- border-right: 1px solid #ddd;
-}
-.tabs-left > .nav-tabs > li > a {
- margin-right: -1px;
- -webkit-border-radius: 4px 0 0 4px;
- -moz-border-radius: 4px 0 0 4px;
- border-radius: 4px 0 0 4px;
-}
-.tabs-left > .nav-tabs > li > a:hover,
-.tabs-left > .nav-tabs > li > a:focus {
- border-color: #eeeeee #dddddd #eeeeee #eeeeee;
-}
-.tabs-left > .nav-tabs .active > a,
-.tabs-left > .nav-tabs .active > a:hover,
-.tabs-left > .nav-tabs .active > a:focus {
- border-color: #ddd transparent #ddd #ddd;
- *border-right-color: #ffffff;
-}
-.tabs-right > .nav-tabs {
- float: right;
- margin-left: 19px;
- border-left: 1px solid #ddd;
-}
-.tabs-right > .nav-tabs > li > a {
- margin-left: -1px;
- -webkit-border-radius: 0 4px 4px 0;
- -moz-border-radius: 0 4px 4px 0;
- border-radius: 0 4px 4px 0;
-}
-.tabs-right > .nav-tabs > li > a:hover,
-.tabs-right > .nav-tabs > li > a:focus {
- border-color: #eeeeee #eeeeee #eeeeee #dddddd;
-}
-.tabs-right > .nav-tabs .active > a,
-.tabs-right > .nav-tabs .active > a:hover,
-.tabs-right > .nav-tabs .active > a:focus {
- border-color: #ddd #ddd #ddd transparent;
- *border-left-color: #ffffff;
-}
-.nav > .disabled > a {
- color: #999999;
-}
-.nav > .disabled > a:hover,
-.nav > .disabled > a:focus {
- text-decoration: none;
- background-color: transparent;
- cursor: default;
-}
-.navbar {
- overflow: visible;
- margin-bottom: 20px;
- *position: relative;
- *z-index: 2;
-}
-.navbar-inner {
- min-height: 40px;
- padding-left: 20px;
- padding-right: 20px;
- background-color: #fafafa;
- background-image: -moz-linear-gradient(top, #ffffff, #f2f2f2);
- background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#ffffff), to(#f2f2f2));
- background-image: -webkit-linear-gradient(top, #ffffff, #f2f2f2);
- background-image: -o-linear-gradient(top, #ffffff, #f2f2f2);
- background-image: linear-gradient(to bottom, #ffffff, #f2f2f2);
- background-repeat: repeat-x;
- filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffffffff', endColorstr='#fff2f2f2', GradientType=0);
- border: 1px solid #d4d4d4;
- -webkit-border-radius: 4px;
- -moz-border-radius: 4px;
- border-radius: 4px;
- -webkit-box-shadow: 0 1px 4px rgba(0, 0, 0, 0.065);
- -moz-box-shadow: 0 1px 4px rgba(0, 0, 0, 0.065);
- box-shadow: 0 1px 4px rgba(0, 0, 0, 0.065);
- *zoom: 1;
-}
-.navbar-inner:before,
-.navbar-inner:after {
- display: table;
- content: "";
- line-height: 0;
-}
-.navbar-inner:after {
- clear: both;
-}
-.navbar .container {
- width: auto;
-}
-.nav-collapse.collapse {
- height: auto;
- overflow: visible;
-}
-.navbar .brand {
- float: left;
- display: block;
- padding: 10px 20px 10px;
- margin-left: -20px;
- font-size: 20px;
- font-weight: 200;
- color: #777777;
- text-shadow: 0 1px 0 #ffffff;
-}
-.navbar .brand:hover,
-.navbar .brand:focus {
- text-decoration: none;
-}
-.navbar-text {
- margin-bottom: 0;
- line-height: 40px;
- color: #777777;
-}
-.navbar-link {
- color: #777777;
-}
-.navbar-link:hover,
-.navbar-link:focus {
- color: #333333;
-}
-.navbar .divider-vertical {
- height: 40px;
- margin: 0 9px;
- border-left: 1px solid #f2f2f2;
- border-right: 1px solid #ffffff;
-}
-.navbar .btn,
-.navbar .btn-group {
- margin-top: 5px;
-}
-.navbar .btn-group .btn,
-.navbar .input-prepend .btn,
-.navbar .input-append .btn,
-.navbar .input-prepend .btn-group,
-.navbar .input-append .btn-group {
- margin-top: 0;
-}
-.navbar-form {
- margin-bottom: 0;
- *zoom: 1;
-}
-.navbar-form:before,
-.navbar-form:after {
- display: table;
- content: "";
- line-height: 0;
-}
-.navbar-form:after {
- clear: both;
-}
-.navbar-form input,
-.navbar-form select,
-.navbar-form .radio,
-.navbar-form .checkbox {
- margin-top: 5px;
-}
-.navbar-form input,
-.navbar-form select,
-.navbar-form .btn {
- display: inline-block;
- margin-bottom: 0;
-}
-.navbar-form input[type="image"],
-.navbar-form input[type="checkbox"],
-.navbar-form input[type="radio"] {
- margin-top: 3px;
-}
-.navbar-form .input-append,
-.navbar-form .input-prepend {
- margin-top: 5px;
- white-space: nowrap;
-}
-.navbar-form .input-append input,
-.navbar-form .input-prepend input {
- margin-top: 0;
-}
-.navbar-search {
- position: relative;
- float: left;
- margin-top: 5px;
- margin-bottom: 0;
-}
-.navbar-search .search-query {
- margin-bottom: 0;
- padding: 4px 14px;
- font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
- font-size: 13px;
- font-weight: normal;
- line-height: 1;
- -webkit-border-radius: 15px;
- -moz-border-radius: 15px;
- border-radius: 15px;
-}
-.navbar-static-top {
- position: static;
- margin-bottom: 0;
-}
-.navbar-static-top .navbar-inner {
- -webkit-border-radius: 0;
- -moz-border-radius: 0;
- border-radius: 0;
-}
-.navbar-fixed-top,
-.navbar-fixed-bottom {
- position: fixed;
- right: 0;
- left: 0;
- z-index: 1030;
- margin-bottom: 0;
-}
-.navbar-fixed-top .navbar-inner,
-.navbar-static-top .navbar-inner {
- border-width: 0 0 1px;
-}
-.navbar-fixed-bottom .navbar-inner {
- border-width: 1px 0 0;
-}
-.navbar-fixed-top .navbar-inner,
-.navbar-fixed-bottom .navbar-inner {
- padding-left: 0;
- padding-right: 0;
- -webkit-border-radius: 0;
- -moz-border-radius: 0;
- border-radius: 0;
-}
-.navbar-static-top .container,
-.navbar-fixed-top .container,
-.navbar-fixed-bottom .container {
- width: 940px;
-}
-.navbar-fixed-top {
- top: 0;
-}
-.navbar-fixed-top .navbar-inner,
-.navbar-static-top .navbar-inner {
- -webkit-box-shadow: 0 1px 10px rgba(0,0,0,.1);
- -moz-box-shadow: 0 1px 10px rgba(0,0,0,.1);
- box-shadow: 0 1px 10px rgba(0,0,0,.1);
-}
-.navbar-fixed-bottom {
- bottom: 0;
-}
-.navbar-fixed-bottom .navbar-inner {
- -webkit-box-shadow: 0 -1px 10px rgba(0,0,0,.1);
- -moz-box-shadow: 0 -1px 10px rgba(0,0,0,.1);
- box-shadow: 0 -1px 10px rgba(0,0,0,.1);
-}
-.navbar .nav {
- position: relative;
- left: 0;
- display: block;
- float: left;
- margin: 0 10px 0 0;
-}
-.navbar .nav.pull-right {
- float: right;
- margin-right: 0;
-}
-.navbar .nav > li {
- float: left;
-}
-.navbar .nav > li > a {
- float: none;
- padding: 10px 15px 10px;
- color: #777777;
- text-decoration: none;
- text-shadow: 0 1px 0 #ffffff;
-}
-.navbar .nav .dropdown-toggle .caret {
- margin-top: 8px;
-}
-.navbar .nav > li > a:focus,
-.navbar .nav > li > a:hover {
- background-color: transparent;
- color: #333333;
- text-decoration: none;
-}
-.navbar .nav > .active > a,
-.navbar .nav > .active > a:hover,
-.navbar .nav > .active > a:focus {
- color: #555555;
- text-decoration: none;
- background-color: #e5e5e5;
- -webkit-box-shadow: inset 0 3px 8px rgba(0, 0, 0, 0.125);
- -moz-box-shadow: inset 0 3px 8px rgba(0, 0, 0, 0.125);
- box-shadow: inset 0 3px 8px rgba(0, 0, 0, 0.125);
-}
-.navbar .btn-navbar {
- display: none;
- float: right;
- padding: 7px 10px;
- margin-left: 5px;
- margin-right: 5px;
- color: #ffffff;
- text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25);
- background-color: #ededed;
- background-image: -moz-linear-gradient(top, #f2f2f2, #e5e5e5);
- background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#f2f2f2), to(#e5e5e5));
- background-image: -webkit-linear-gradient(top, #f2f2f2, #e5e5e5);
- background-image: -o-linear-gradient(top, #f2f2f2, #e5e5e5);
- background-image: linear-gradient(to bottom, #f2f2f2, #e5e5e5);
- background-repeat: repeat-x;
- filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#fff2f2f2', endColorstr='#ffe5e5e5', GradientType=0);
- border-color: #e5e5e5 #e5e5e5 #bfbfbf;
- border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
- *background-color: #e5e5e5;
- /* Darken IE7 buttons by default so they stand out more given they won't have borders */
-
- filter: progid:DXImageTransform.Microsoft.gradient(enabled = false);
- -webkit-box-shadow: inset 0 1px 0 rgba(255,255,255,.1), 0 1px 0 rgba(255,255,255,.075);
- -moz-box-shadow: inset 0 1px 0 rgba(255,255,255,.1), 0 1px 0 rgba(255,255,255,.075);
- box-shadow: inset 0 1px 0 rgba(255,255,255,.1), 0 1px 0 rgba(255,255,255,.075);
-}
-.navbar .btn-navbar:hover,
-.navbar .btn-navbar:focus,
-.navbar .btn-navbar:active,
-.navbar .btn-navbar.active,
-.navbar .btn-navbar.disabled,
-.navbar .btn-navbar[disabled] {
- color: #ffffff;
- background-color: #e5e5e5;
- *background-color: #d9d9d9;
-}
-.navbar .btn-navbar:active,
-.navbar .btn-navbar.active {
- background-color: #cccccc \9;
-}
-.navbar .btn-navbar .icon-bar {
- display: block;
- width: 18px;
- height: 2px;
- background-color: #f5f5f5;
- -webkit-border-radius: 1px;
- -moz-border-radius: 1px;
- border-radius: 1px;
- -webkit-box-shadow: 0 1px 0 rgba(0, 0, 0, 0.25);
- -moz-box-shadow: 0 1px 0 rgba(0, 0, 0, 0.25);
- box-shadow: 0 1px 0 rgba(0, 0, 0, 0.25);
-}
-.btn-navbar .icon-bar + .icon-bar {
- margin-top: 3px;
-}
-.navbar .nav > li > .dropdown-menu:before {
- content: '';
- display: inline-block;
- border-left: 7px solid transparent;
- border-right: 7px solid transparent;
- border-bottom: 7px solid #ccc;
- border-bottom-color: rgba(0, 0, 0, 0.2);
- position: absolute;
- top: -7px;
- left: 9px;
-}
-.navbar .nav > li > .dropdown-menu:after {
- content: '';
- display: inline-block;
- border-left: 6px solid transparent;
- border-right: 6px solid transparent;
- border-bottom: 6px solid #ffffff;
- position: absolute;
- top: -6px;
- left: 10px;
-}
-.navbar-fixed-bottom .nav > li > .dropdown-menu:before {
- border-top: 7px solid #ccc;
- border-top-color: rgba(0, 0, 0, 0.2);
- border-bottom: 0;
- bottom: -7px;
- top: auto;
-}
-.navbar-fixed-bottom .nav > li > .dropdown-menu:after {
- border-top: 6px solid #ffffff;
- border-bottom: 0;
- bottom: -6px;
- top: auto;
-}
-.navbar .nav li.dropdown > a:hover .caret,
-.navbar .nav li.dropdown > a:focus .caret {
- border-top-color: #333333;
- border-bottom-color: #333333;
-}
-.navbar .nav li.dropdown.open > .dropdown-toggle,
-.navbar .nav li.dropdown.active > .dropdown-toggle,
-.navbar .nav li.dropdown.open.active > .dropdown-toggle {
- background-color: #e5e5e5;
- color: #555555;
-}
-.navbar .nav li.dropdown > .dropdown-toggle .caret {
- border-top-color: #777777;
- border-bottom-color: #777777;
-}
-.navbar .nav li.dropdown.open > .dropdown-toggle .caret,
-.navbar .nav li.dropdown.active > .dropdown-toggle .caret,
-.navbar .nav li.dropdown.open.active > .dropdown-toggle .caret {
- border-top-color: #555555;
- border-bottom-color: #555555;
-}
-.navbar .pull-right > li > .dropdown-menu,
-.navbar .nav > li > .dropdown-menu.pull-right {
- left: auto;
- right: 0;
-}
-.navbar .pull-right > li > .dropdown-menu:before,
-.navbar .nav > li > .dropdown-menu.pull-right:before {
- left: auto;
- right: 12px;
-}
-.navbar .pull-right > li > .dropdown-menu:after,
-.navbar .nav > li > .dropdown-menu.pull-right:after {
- left: auto;
- right: 13px;
-}
-.navbar .pull-right > li > .dropdown-menu .dropdown-menu,
-.navbar .nav > li > .dropdown-menu.pull-right .dropdown-menu {
- left: auto;
- right: 100%;
- margin-left: 0;
- margin-right: -1px;
- -webkit-border-radius: 6px 0 6px 6px;
- -moz-border-radius: 6px 0 6px 6px;
- border-radius: 6px 0 6px 6px;
-}
-.navbar-inverse .navbar-inner {
- background-color: #1b1b1b;
- background-image: -moz-linear-gradient(top, #222222, #111111);
- background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#222222), to(#111111));
- background-image: -webkit-linear-gradient(top, #222222, #111111);
- background-image: -o-linear-gradient(top, #222222, #111111);
- background-image: linear-gradient(to bottom, #222222, #111111);
- background-repeat: repeat-x;
- filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff222222', endColorstr='#ff111111', GradientType=0);
- border-color: #252525;
-}
-.navbar-inverse .brand,
-.navbar-inverse .nav > li > a {
- color: #999999;
- text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25);
-}
-.navbar-inverse .brand:hover,
-.navbar-inverse .nav > li > a:hover,
-.navbar-inverse .brand:focus,
-.navbar-inverse .nav > li > a:focus {
- color: #ffffff;
-}
-.navbar-inverse .brand {
- color: #999999;
-}
-.navbar-inverse .navbar-text {
- color: #999999;
-}
-.navbar-inverse .nav > li > a:focus,
-.navbar-inverse .nav > li > a:hover {
- background-color: transparent;
- color: #ffffff;
-}
-.navbar-inverse .nav .active > a,
-.navbar-inverse .nav .active > a:hover,
-.navbar-inverse .nav .active > a:focus {
- color: #ffffff;
- background-color: #111111;
-}
-.navbar-inverse .navbar-link {
- color: #999999;
-}
-.navbar-inverse .navbar-link:hover,
-.navbar-inverse .navbar-link:focus {
- color: #ffffff;
-}
-.navbar-inverse .divider-vertical {
- border-left-color: #111111;
- border-right-color: #222222;
-}
-.navbar-inverse .nav li.dropdown.open > .dropdown-toggle,
-.navbar-inverse .nav li.dropdown.active > .dropdown-toggle,
-.navbar-inverse .nav li.dropdown.open.active > .dropdown-toggle {
- background-color: #111111;
- color: #ffffff;
-}
-.navbar-inverse .nav li.dropdown > a:hover .caret,
-.navbar-inverse .nav li.dropdown > a:focus .caret {
- border-top-color: #ffffff;
- border-bottom-color: #ffffff;
-}
-.navbar-inverse .nav li.dropdown > .dropdown-toggle .caret {
- border-top-color: #999999;
- border-bottom-color: #999999;
-}
-.navbar-inverse .nav li.dropdown.open > .dropdown-toggle .caret,
-.navbar-inverse .nav li.dropdown.active > .dropdown-toggle .caret,
-.navbar-inverse .nav li.dropdown.open.active > .dropdown-toggle .caret {
- border-top-color: #ffffff;
- border-bottom-color: #ffffff;
-}
-.navbar-inverse .navbar-search .search-query {
- color: #ffffff;
- background-color: #515151;
- border-color: #111111;
- -webkit-box-shadow: inset 0 1px 2px rgba(0,0,0,.1), 0 1px 0 rgba(255,255,255,.15);
- -moz-box-shadow: inset 0 1px 2px rgba(0,0,0,.1), 0 1px 0 rgba(255,255,255,.15);
- box-shadow: inset 0 1px 2px rgba(0,0,0,.1), 0 1px 0 rgba(255,255,255,.15);
- -webkit-transition: none;
- -moz-transition: none;
- -o-transition: none;
- transition: none;
-}
-.navbar-inverse .navbar-search .search-query:-moz-placeholder {
- color: #cccccc;
-}
-.navbar-inverse .navbar-search .search-query:-ms-input-placeholder {
- color: #cccccc;
-}
-.navbar-inverse .navbar-search .search-query::-webkit-input-placeholder {
- color: #cccccc;
-}
-.navbar-inverse .navbar-search .search-query:focus,
-.navbar-inverse .navbar-search .search-query.focused {
- padding: 5px 15px;
- color: #333333;
- text-shadow: 0 1px 0 #ffffff;
- background-color: #ffffff;
- border: 0;
- -webkit-box-shadow: 0 0 3px rgba(0, 0, 0, 0.15);
- -moz-box-shadow: 0 0 3px rgba(0, 0, 0, 0.15);
- box-shadow: 0 0 3px rgba(0, 0, 0, 0.15);
- outline: 0;
-}
-.navbar-inverse .btn-navbar {
- color: #ffffff;
- text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25);
- background-color: #0e0e0e;
- background-image: -moz-linear-gradient(top, #151515, #040404);
- background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#151515), to(#040404));
- background-image: -webkit-linear-gradient(top, #151515, #040404);
- background-image: -o-linear-gradient(top, #151515, #040404);
- background-image: linear-gradient(to bottom, #151515, #040404);
- background-repeat: repeat-x;
- filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff151515', endColorstr='#ff040404', GradientType=0);
- border-color: #040404 #040404 #000000;
- border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
- *background-color: #040404;
- /* Darken IE7 buttons by default so they stand out more given they won't have borders */
-
- filter: progid:DXImageTransform.Microsoft.gradient(enabled = false);
-}
-.navbar-inverse .btn-navbar:hover,
-.navbar-inverse .btn-navbar:focus,
-.navbar-inverse .btn-navbar:active,
-.navbar-inverse .btn-navbar.active,
-.navbar-inverse .btn-navbar.disabled,
-.navbar-inverse .btn-navbar[disabled] {
- color: #ffffff;
- background-color: #040404;
- *background-color: #000000;
-}
-.navbar-inverse .btn-navbar:active,
-.navbar-inverse .btn-navbar.active {
- background-color: #000000 \9;
-}
-/*@import "bootstrap/breadcrumbs.less";*/
-/*@import "bootstrap/pagination.less";*/
-/*@import "bootstrap/pager.less";*/
-.modal-backdrop {
- position: fixed;
- top: 0;
- right: 0;
- bottom: 0;
- left: 0;
- z-index: 1040;
- background-color: #000000;
-}
-.modal-backdrop.fade {
- opacity: 0;
-}
-.modal-backdrop,
-.modal-backdrop.fade.in {
- opacity: 0.8;
- filter: alpha(opacity=80);
-}
-.modal {
- position: fixed;
- top: 10%;
- left: 50%;
- z-index: 1050;
- width: 560px;
- margin-left: -280px;
- background-color: #ffffff;
- border: 1px solid #999;
- border: 1px solid rgba(0, 0, 0, 0.3);
- *border: 1px solid #999;
- /* IE6-7 */
-
- -webkit-border-radius: 6px;
- -moz-border-radius: 6px;
- border-radius: 6px;
- -webkit-box-shadow: 0 3px 7px rgba(0, 0, 0, 0.3);
- -moz-box-shadow: 0 3px 7px rgba(0, 0, 0, 0.3);
- box-shadow: 0 3px 7px rgba(0, 0, 0, 0.3);
- -webkit-background-clip: padding-box;
- -moz-background-clip: padding-box;
- background-clip: padding-box;
- outline: none;
-}
-.modal.fade {
- -webkit-transition: opacity .3s linear, top .3s ease-out;
- -moz-transition: opacity .3s linear, top .3s ease-out;
- -o-transition: opacity .3s linear, top .3s ease-out;
- transition: opacity .3s linear, top .3s ease-out;
- top: -25%;
-}
-.modal.fade.in {
- top: 10%;
-}
-.modal-header {
- padding: 9px 15px;
- border-bottom: 1px solid #eee;
-}
-.modal-header .close {
- margin-top: 2px;
-}
-.modal-header h3 {
- margin: 0;
- line-height: 30px;
-}
-.modal-body {
- position: relative;
- overflow-y: auto;
- max-height: 400px;
- padding: 15px;
-}
-.modal-form {
- margin-bottom: 0;
-}
-.modal-footer {
- padding: 14px 15px 15px;
- margin-bottom: 0;
- text-align: right;
- background-color: #f5f5f5;
- border-top: 1px solid #ddd;
- -webkit-border-radius: 0 0 6px 6px;
- -moz-border-radius: 0 0 6px 6px;
- border-radius: 0 0 6px 6px;
- -webkit-box-shadow: inset 0 1px 0 #ffffff;
- -moz-box-shadow: inset 0 1px 0 #ffffff;
- box-shadow: inset 0 1px 0 #ffffff;
- *zoom: 1;
-}
-.modal-footer:before,
-.modal-footer:after {
- display: table;
- content: "";
- line-height: 0;
-}
-.modal-footer:after {
- clear: both;
-}
-.modal-footer .btn + .btn {
- margin-left: 5px;
- margin-bottom: 0;
-}
-.modal-footer .btn-group .btn + .btn {
- margin-left: -1px;
-}
-.modal-footer .btn-block + .btn-block {
- margin-left: 0;
-}
-/*@import "bootstrap/tooltip.less";*/
-/*@import "bootstrap/popovers.less";*/
-/*@import "bootstrap/thumbnails.less";*/
-/*@import "bootstrap/media.less";*/
-/*@import "bootstrap/labels-badges.less";*/
-/*@import "bootstrap/progress-bars.less";*/
-/*@import "bootstrap/accordion.less";*/
-/*@import "bootstrap/carousel.less";*/
-/*@import "bootstrap/hero-unit.less";*/
-.pull-right {
- float: right;
-}
-.pull-left {
- float: left;
-}
-.hide {
- display: none;
-}
-.show {
- display: block;
-}
-.invisible {
- visibility: hidden;
-}
-.affix {
- position: fixed;
-}
-/* http://meyerweb.com/eric/tools/css/reset/
- v2.0 | 20110126
- License: none (public domain)
-*/
-html,
-body,
-div,
-span,
-applet,
-object,
-iframe,
-h1,
-h2,
-h3,
-h4,
-h5,
-h6,
-p,
-blockquote,
-pre,
-a,
-abbr,
-acronym,
-address,
-big,
-cite,
-code,
-del,
-dfn,
-em,
-img,
-ins,
-kbd,
-q,
-s,
-samp,
-small,
-strike,
-strong,
-sub,
-sup,
-tt,
-var,
-b,
-u,
-i,
-center,
-dl,
-dt,
-dd,
-ol,
-ul,
-li,
-fieldset,
-form,
-label,
-legend,
-table,
-caption,
-tbody,
-tfoot,
-thead,
-tr,
-th,
-td,
-article,
-aside,
-canvas,
-details,
-embed,
-figure,
-figcaption,
-footer,
-header,
-hgroup,
-menu,
-nav,
-output,
-ruby,
-section,
-summary,
-time,
-mark,
-audio,
-video {
- margin: 0;
- padding: 0;
- border: 0;
- font-size: 100%;
- font: inherit;
- vertical-align: baseline;
-}
-/* HTML5 display-role reset for older browsers */
-article,
-aside,
-deta
<TRUNCATED>
[30/30] apex-malhar git commit: Renamed demos to examples. Packages
and artifactid names are changed as suggested.
Posted by th...@apache.org.
Renamed demos to examples.
Packages and artifactid names are changed as suggested.
Project: http://git-wip-us.apache.org/repos/asf/apex-malhar/repo
Commit: http://git-wip-us.apache.org/repos/asf/apex-malhar/commit/d5bf96ca
Tree: http://git-wip-us.apache.org/repos/asf/apex-malhar/tree/d5bf96ca
Diff: http://git-wip-us.apache.org/repos/asf/apex-malhar/diff/d5bf96ca
Branch: refs/heads/master
Commit: d5bf96cac175cfd7e2d4a0effad1f057c0e8c604
Parents: 4b36bf3
Author: Apex Dev <de...@apex.apache.org>
Authored: Fri Feb 24 16:27:28 2017 -0800
Committer: Lakshmi Prasanna Velineni <la...@datatorrent.com>
Committed: Mon Mar 6 22:13:24 2017 -0800
----------------------------------------------------------------------
README.md | 4 +-
demos/distributedistinct/pom.xml | 41 -
.../src/assemble/appPackage.xml | 59 -
.../demos/distributeddistinct/Application.java | 69 -
.../distributeddistinct/CountVerifier.java | 103 -
.../IntegerUniqueValueCountAppender.java | 95 -
.../RandomKeyValGenerator.java | 186 -
.../StatefulApplication.java | 76 -
.../UniqueValueCountAppender.java | 236 -
.../src/main/resources/META-INF/properties.xml | 29 -
.../src/site/conf/my-app-conf1.xml | 27 -
.../distributeddistinct/ApplicationTest.java | 32 -
.../DistributedDistinctTest.java | 200 -
.../StatefulApplicationTest.java | 78 -
.../StatefulUniqueCountTest.java | 248 -
.../src/test/resources/log4j.properties | 43 -
demos/echoserver/pom.xml | 38 -
demos/echoserver/src/assemble/appPackage.xml | 59 -
.../demos/echoserver/Application.java | 43 -
.../datatorrent/demos/echoserver/Message.java | 30 -
.../demos/echoserver/MessageReceiver.java | 156 -
.../demos/echoserver/MessageResponder.java | 81 -
.../demos/echoserver/NetworkManager.java | 249 -
.../src/main/resources/META-INF/properties.xml | 38 -
demos/echoserver/src/site/conf/my-app-conf1.xml | 27 -
.../demos/echoserver/ApplicationTest.java | 53 -
.../src/test/resources/log4j.properties | 43 -
demos/frauddetect/pom.xml | 49 -
demos/frauddetect/src/assemble/appPackage.xml | 59 -
.../demos/frauddetect/Application.java | 151 -
.../demos/frauddetect/AverageAlertData.java | 37 -
.../frauddetect/AverageAlertingOperator.java | 185 -
.../frauddetect/BankIdNumberAlertData.java | 36 -
.../demos/frauddetect/BankIdNumberKey.java | 65 -
.../BankIdNumberSamplerOperator.java | 216 -
.../demos/frauddetect/CreditCardAlertData.java | 38 -
.../CreditCardAmountSamplerOperator.java | 206 -
.../demos/frauddetect/CreditCardData.java | 35 -
.../demos/frauddetect/MerchantKey.java | 128 -
.../frauddetect/MerchantQueryInputHandler.java | 69 -
.../demos/frauddetect/MerchantTransaction.java | 202 -
.../MerchantTransactionBucketOperator.java | 148 -
.../MerchantTransactionGenerator.java | 210 -
.../MerchantTransactionInputHandler.java | 116 -
.../frauddetect/SlidingWindowSumKeyVal.java | 99 -
.../frauddetect/SlidingWindowSumObject.java | 53 -
.../frauddetect/TransactionStatsAggregator.java | 99 -
.../demos/frauddetect/TransactionStatsData.java | 36 -
.../operator/HdfsStringOutputOperator.java | 72 -
.../operator/MongoDBOutputOperator.java | 188 -
.../demos/frauddetect/util/JsonUtils.java | 38 -
.../src/main/resources/META-INF/properties.xml | 167 -
.../frauddetect/src/site/conf/my-app-conf1.xml | 27 -
.../frauddetect/FrauddetectApplicationTest.java | 50 -
.../src/test/resources/dt-site-frauddetect.xml | 173 -
.../src/test/resources/log4j.properties | 43 -
demos/highlevelapi/pom.xml | 141 -
demos/highlevelapi/src/assemble/appPackage.xml | 59 -
.../malhar/stream/sample/MinimalWordCount.java | 128 -
.../malhar/stream/sample/WindowedWordCount.java | 290 --
.../stream/sample/complete/AutoComplete.java | 324 --
.../sample/complete/CompletionCandidate.java | 89 -
.../stream/sample/complete/PojoEvent.java | 46 -
.../sample/complete/StreamingWordExtract.java | 162 -
.../stream/sample/complete/TopNByKey.java | 120 -
.../sample/complete/TopWikipediaSessions.java | 347 --
.../stream/sample/complete/TrafficRoutes.java | 523 --
.../sample/complete/TwitterAutoComplete.java | 254 -
.../sample/cookbook/CombinePerKeyExamples.java | 285 --
.../stream/sample/cookbook/DeDupExample.java | 127 -
.../stream/sample/cookbook/InputPojo.java | 78 -
.../sample/cookbook/MaxPerKeyExamples.java | 205 -
.../stream/sample/cookbook/OutputPojo.java | 56 -
.../stream/sample/cookbook/TriggerExample.java | 577 ---
.../src/main/resources/META-INF/properties.xml | 141 -
.../stream/sample/MinimalWordCountTest.java | 61 -
.../stream/sample/WindowedWordCountTest.java | 91 -
.../sample/complete/AutoCompleteTest.java | 62 -
.../complete/StreamingWordExtractTest.java | 144 -
.../complete/TopWikipediaSessionsTest.java | 74 -
.../sample/complete/TrafficRoutesTest.java | 67 -
.../complete/TwitterAutoCompleteTest.java | 66 -
.../cookbook/CombinePerKeyExamplesTest.java | 56 -
.../sample/cookbook/DeDupExampleTest.java | 59 -
.../sample/cookbook/MaxPerKeyExamplesTest.java | 210 -
.../src/test/resources/data/word.txt | 2 -
.../src/test/resources/log4j.properties | 45 -
.../src/test/resources/sampletweets.txt | 44 -
.../src/test/resources/wordcount/word.txt | 8 -
demos/iteration/pom.xml | 37 -
demos/iteration/src/assemble/appPackage.xml | 59 -
.../demos/iteration/Application.java | 171 -
.../demos/iteration/package-info.java | 22 -
.../src/main/resources/META-INF/properties.xml | 45 -
.../demos/iteration/ApplicationTest.java | 86 -
.../src/test/resources/log4j.properties | 43 -
demos/machinedata/pom.xml | 66 -
demos/machinedata/src/assemble/appPackage.xml | 59 -
demos/machinedata/src/main/html/global.js | 269 -
demos/machinedata/src/main/html/index.php | 263 -
demos/machinedata/src/main/html/json.php | 96 -
demos/machinedata/src/main/html/malhar.css | 4688 ------------------
.../demos/machinedata/Application.java | 84 -
.../demos/machinedata/DimensionGenerator.java | 120 -
.../demos/machinedata/InputReceiver.java | 523 --
.../demos/machinedata/data/AverageData.java | 107 -
.../demos/machinedata/data/MachineInfo.java | 158 -
.../demos/machinedata/data/MachineKey.java | 381 --
.../demos/machinedata/data/ResourceType.java | 66 -
.../operator/CalculatorOperator.java | 277 --
.../operator/MachineInfoAveragingOperator.java | 215 -
...chineInfoAveragingPrerequisitesOperator.java | 87 -
.../operator/MachineInfoAveragingUnifier.java | 91 -
.../demos/machinedata/util/Combinatorics.java | 101 -
.../demos/machinedata/util/DataTable.java | 75 -
.../src/main/resources/META-INF/properties.xml | 139 -
.../machinedata/src/site/conf/my-app-conf1.xml | 27 -
.../machinedata/CalculatorOperatorTest.java | 196 -
.../src/test/resources/log4j.properties | 43 -
demos/mobile/pom.xml | 64 -
demos/mobile/src/assemble/appPackage.xml | 59 -
.../datatorrent/demos/mobile/Application.java | 172 -
.../demos/mobile/PhoneEntryOperator.java | 124 -
.../demos/mobile/PhoneMovementGenerator.java | 335 --
.../demos/mobile/doc-files/Mobile.png | Bin 24390 -> 0 bytes
.../datatorrent/demos/mobile/package-info.java | 22 -
.../src/main/resources/META-INF/properties.xml | 82 -
demos/mobile/src/site/conf/my-app-conf1.xml | 27 -
.../demos/mobile/ApplicationTest.java | 117 -
.../src/test/resources/dt-site-mobile.xml | 87 -
.../mobile/src/test/resources/log4j.properties | 43 -
demos/mrmonitor/pom.xml | 64 -
demos/mrmonitor/src/assemble/appPackage.xml | 59 -
.../demos/mrmonitor/Application.java | 56 -
.../datatorrent/demos/mrmonitor/Constants.java | 48 -
.../demos/mrmonitor/MRJobStatusOperator.java | 623 ---
.../mrmonitor/MRMonitoringApplication.java | 93 -
.../demos/mrmonitor/MRStatusObject.java | 501 --
.../com/datatorrent/demos/mrmonitor/MRUtil.java | 99 -
.../demos/mrmonitor/MapToMRObjectOperator.java | 89 -
.../src/main/resources/META-INF/properties.xml | 63 -
.../src/main/resources/mrdebugger.html | 237 -
demos/mrmonitor/src/site/conf/my-app-conf1.xml | 27 -
.../mrmonitor/MrMonitoringApplicationTest.java | 66 -
.../src/test/resources/dt-site-monitoring.xml | 63 -
.../src/test/resources/log4j.properties | 43 -
demos/mroperator/pom.xml | 56 -
demos/mroperator/src/assemble/appPackage.xml | 59 -
.../demos/mroperator/DateWritable.java | 80 -
.../mroperator/HdfsKeyValOutputOperator.java | 41 -
.../mroperator/InvertedIndexApplication.java | 45 -
.../demos/mroperator/LineIndexer.java | 120 -
.../demos/mroperator/LogCountsPerHour.java | 187 -
.../demos/mroperator/LogsCountApplication.java | 46 -
.../demos/mroperator/MapOperator.java | 414 --
.../demos/mroperator/MapReduceApplication.java | 114 -
.../mroperator/NewWordCountApplication.java | 44 -
.../demos/mroperator/OutputCollectorImpl.java | 94 -
.../demos/mroperator/ReduceOperator.java | 189 -
.../demos/mroperator/ReporterImpl.java | 119 -
.../datatorrent/demos/mroperator/WordCount.java | 104 -
.../src/main/resources/META-INF/properties.xml | 88 -
demos/mroperator/src/site/conf/my-app-conf1.xml | 27 -
.../demos/mroperator/MapOperatorTest.java | 159 -
.../demos/mroperator/ReduceOperatorTest.java | 74 -
.../mroperator/WordCountMRApplicationTest.java | 74 -
.../src/test/resources/log4j.properties | 43 -
demos/pi/pom.xml | 45 -
demos/pi/src/assemble/appPackage.xml | 59 -
.../com/datatorrent/demos/pi/Application.java | 92 -
.../demos/pi/ApplicationAppData.java | 135 -
.../demos/pi/ApplicationWithScript.java | 112 -
.../com/datatorrent/demos/pi/Calculator.java | 81 -
.../datatorrent/demos/pi/NamedValueList.java | 94 -
.../demos/pi/PiCalculateOperator.java | 90 -
.../demos/pi/doc-files/Application.gif | Bin 6791 -> 0 bytes
.../demos/pi/doc-files/ApplicationScript.gif | Bin 6687 -> 0 bytes
.../com/datatorrent/demos/pi/package-info.java | 22 -
.../src/main/resources/META-INF/properties.xml | 109 -
.../pi/src/main/resources/PiDemoDataSchema.json | 3 -
demos/pi/src/main/resources/app/PiJsonDemo.json | 52 -
.../datatorrent/demos/pi/ApplicationTest.java | 42 -
.../demos/pi/ApplicationWithScriptTest.java | 35 -
.../datatorrent/demos/pi/CalculatorTest.java | 41 -
demos/pi/src/test/resources/dt-site-pi.xml | 40 -
.../pi/src/test/resources/dt-site-pilibrary.xml | 45 -
demos/pi/src/test/resources/log4j.properties | 43 -
demos/pom.xml | 231 -
demos/r/pom.xml | 83 -
demos/r/src/assemble/appPackage.xml | 59 -
.../demos/r/oldfaithful/FaithfulKey.java | 55 -
.../demos/r/oldfaithful/FaithfulRScript.java | 111 -
.../demos/r/oldfaithful/InputGenerator.java | 145 -
.../r/oldfaithful/OldFaithfulApplication.java | 75 -
.../src/main/resources/META-INF/properties.xml | 41 -
.../demos/r/oldfaithful/eruptionModel.R | 60 -
.../oldfaithful/OldFaithfulApplicationTest.java | 51 -
.../src/test/resources/dt-site-oldfaithful.xml | 41 -
demos/r/src/test/resources/log4j.properties | 43 -
demos/sql/pom.xml | 102 -
demos/sql/src/assemble/appPackage.xml | 59 -
.../sql/sample/FusionStyleSQLApplication.java | 91 -
.../sql/sample/PureStyleSQLApplication.java | 68 -
.../sql/sample/SQLApplicationWithAPI.java | 48 -
.../sql/sample/SQLApplicationWithModelFile.java | 53 -
.../properties-FusionStyleSQLApplication.xml | 65 -
.../properties-PureStyleSQLApplication.xml | 65 -
.../properties-SQLApplicationWithAPI.xml | 43 -
.../properties-SQLApplicationWithModelFile.xml | 32 -
.../src/main/resources/META-INF/properties.xml | 41 -
.../main/resources/model/model_file_csv.json | 27 -
.../sample/FusionStyleSQLApplicationTest.java | 121 -
.../sql/sample/PureStyleSQLApplicationTest.java | 155 -
.../sql/sample/SQLApplicationWithAPITest.java | 92 -
.../sample/SQLApplicationWithModelFileTest.java | 113 -
demos/sql/src/test/resources/input.csv | 6 -
demos/sql/src/test/resources/log4j.properties | 50 -
demos/twitter/pom.xml | 101 -
demos/twitter/src/assemble/appPackage.xml | 59 -
.../twitter/KinesisHashtagsApplication.java | 236 -
.../demos/twitter/SlidingContainer.java | 70 -
.../demos/twitter/TwitterDumpApplication.java | 117 -
.../twitter/TwitterDumpHBaseApplication.java | 81 -
.../twitter/TwitterStatusHashtagExtractor.java | 52 -
.../twitter/TwitterStatusURLExtractor.java | 56 -
.../twitter/TwitterStatusWordExtractor.java | 62 -
.../twitter/TwitterTopCounterApplication.java | 222 -
.../twitter/TwitterTopWordsApplication.java | 71 -
.../TwitterTrendingHashtagsApplication.java | 166 -
.../com/datatorrent/demos/twitter/URLSerDe.java | 74 -
.../demos/twitter/WindowedTopCounter.java | 282 --
.../demos/twitter/doc-files/Application.gif | Bin 8775 -> 0 bytes
.../datatorrent/demos/twitter/package-info.java | 22 -
.../META-INF/properties-TwitterKinesisDemo.xml | 52 -
.../src/main/resources/META-INF/properties.xml | 121 -
demos/twitter/src/main/resources/mysql.sql | 35 -
.../src/main/resources/top_urls.tplg.properties | 48 -
.../resources/twitterHashTagDataSchema.json | 4 -
.../main/resources/twitterURLDataSchema.json | 4 -
.../main/resources/twitterWordDataSchema.json | 4 -
.../twitter/TwitterDumpApplicationTest.java | 48 -
.../demos/twitter/TwitterTopCounterTest.java | 45 -
.../demos/twitter/TwitterTopWordsTest.java | 48 -
.../test/resources/dt-site-rollingtopwords.xml | 73 -
.../twitter/src/test/resources/log4j.properties | 43 -
demos/uniquecount/pom.xml | 50 -
demos/uniquecount/src/assemble/appPackage.xml | 59 -
.../demos/uniquecount/Application.java | 90 -
.../demos/uniquecount/CountVerifier.java | 107 -
.../demos/uniquecount/RandomDataGenerator.java | 115 -
.../demos/uniquecount/RandomKeyValues.java | 153 -
.../demos/uniquecount/RandomKeysGenerator.java | 153 -
.../uniquecount/UniqueKeyValCountDemo.java | 64 -
.../demos/uniquecount/package-info.java | 22 -
.../src/main/resources/META-INF/properties.xml | 29 -
.../uniquecount/src/site/conf/my-app-conf1.xml | 27 -
.../demos/uniquecount/ApplicationTest.java | 38 -
.../demos/uniquecount/UniqueKeyValDemoTest.java | 38 -
.../src/test/resources/log4j.properties | 43 -
demos/wordcount/pom.xml | 49 -
demos/wordcount/src/assemble/appPackage.xml | 59 -
.../demos/wordcount/Application.java | 108 -
.../wordcount/ApplicationWithQuerySupport.java | 133 -
.../demos/wordcount/FileWordCount.java | 349 --
.../datatorrent/demos/wordcount/LineReader.java | 118 -
.../com/datatorrent/demos/wordcount/WCPair.java | 62 -
.../demos/wordcount/WindowWordCount.java | 94 -
.../demos/wordcount/WordCountInputOperator.java | 109 -
.../demos/wordcount/WordCountWriter.java | 109 -
.../datatorrent/demos/wordcount/WordReader.java | 98 -
.../wordcount/doc-files/UniqueWordCounter.jpg | Bin 65015 -> 0 bytes
.../demos/wordcount/package-info.java | 22 -
.../src/main/resources/META-INF/properties.xml | 98 -
.../src/main/resources/WordDataSchema.json | 4 -
.../datatorrent/demos/wordcount/samplefile.txt | 1 -
.../wordcount/src/main/resources/samplefile.txt | 2 -
demos/wordcount/src/site/conf/my-app-conf1.xml | 27 -
.../demos/wordcount/ApplicationTest.java | 51 -
.../src/test/resources/dt-site-wordcount.xml | 37 -
.../src/test/resources/log4j.properties | 43 -
demos/yahoofinance/pom.xml | 65 -
demos/yahoofinance/src/assemble/appPackage.xml | 59 -
.../yahoofinance/ApplicationWithDerbySQL.java | 80 -
.../demos/yahoofinance/StockTickInput.java | 192 -
.../yahoofinance/YahooFinanceApplication.java | 365 --
.../YahooFinanceCSVInputOperator.java | 160 -
.../yahoofinance/doc-files/Application.gif | Bin 12457 -> 0 bytes
.../demos/yahoofinance/package-info.java | 22 -
.../src/main/resources/META-INF/properties.xml | 81 -
.../demos/yahoofinance/ApplicationTest.java | 46 -
.../ApplicationWithDerbySQLTest.java | 50 -
.../src/test/resources/alert_create.json | 24 -
.../src/test/resources/log4j.properties | 43 -
docs/CodingConventionsAndStyle.md | 2 +-
docs/index.md | 2 +-
docs/operators/windowedOperator.md | 2 +-
examples/distributedistinct/pom.xml | 41 +
.../src/assemble/appPackage.xml | 59 +
.../distributeddistinct/Application.java | 69 +
.../distributeddistinct/CountVerifier.java | 103 +
.../IntegerUniqueValueCountAppender.java | 95 +
.../RandomKeyValGenerator.java | 186 +
.../StatefulApplication.java | 76 +
.../UniqueValueCountAppender.java | 236 +
.../src/main/resources/META-INF/properties.xml | 29 +
.../distributeddistinct/ApplicationTest.java | 32 +
.../DistributedDistinctTest.java | 200 +
.../StatefulApplicationTest.java | 78 +
.../StatefulUniqueCountTest.java | 248 +
.../src/test/resources/log4j.properties | 43 +
examples/echoserver/pom.xml | 38 +
examples/echoserver/src/assemble/appPackage.xml | 59 +
.../apex/examples/echoserver/Application.java | 43 +
.../apex/examples/echoserver/Message.java | 30 +
.../examples/echoserver/MessageReceiver.java | 156 +
.../examples/echoserver/MessageResponder.java | 81 +
.../examples/echoserver/NetworkManager.java | 249 +
.../src/main/resources/META-INF/properties.xml | 38 +
.../examples/echoserver/ApplicationTest.java | 53 +
.../src/test/resources/log4j.properties | 43 +
examples/frauddetect/pom.xml | 49 +
.../frauddetect/src/assemble/appPackage.xml | 59 +
.../apex/examples/frauddetect/Application.java | 151 +
.../examples/frauddetect/AverageAlertData.java | 37 +
.../frauddetect/AverageAlertingOperator.java | 186 +
.../frauddetect/BankIdNumberAlertData.java | 36 +
.../examples/frauddetect/BankIdNumberKey.java | 65 +
.../BankIdNumberSamplerOperator.java | 218 +
.../frauddetect/CreditCardAlertData.java | 38 +
.../CreditCardAmountSamplerOperator.java | 206 +
.../examples/frauddetect/CreditCardData.java | 35 +
.../apex/examples/frauddetect/MerchantKey.java | 128 +
.../frauddetect/MerchantQueryInputHandler.java | 69 +
.../frauddetect/MerchantTransaction.java | 202 +
.../MerchantTransactionBucketOperator.java | 148 +
.../MerchantTransactionGenerator.java | 210 +
.../MerchantTransactionInputHandler.java | 116 +
.../frauddetect/SlidingWindowSumKeyVal.java | 99 +
.../frauddetect/SlidingWindowSumObject.java | 53 +
.../frauddetect/TransactionStatsAggregator.java | 99 +
.../frauddetect/TransactionStatsData.java | 36 +
.../operator/HdfsStringOutputOperator.java | 72 +
.../operator/MongoDBOutputOperator.java | 188 +
.../examples/frauddetect/util/JsonUtils.java | 38 +
.../src/main/resources/META-INF/properties.xml | 167 +
.../frauddetect/FrauddetectApplicationTest.java | 50 +
.../src/test/resources/dt-site-frauddetect.xml | 173 +
.../src/test/resources/log4j.properties | 43 +
examples/highlevelapi/pom.xml | 141 +
.../highlevelapi/src/assemble/appPackage.xml | 59 +
.../malhar/stream/sample/MinimalWordCount.java | 128 +
.../malhar/stream/sample/WindowedWordCount.java | 290 ++
.../stream/sample/complete/AutoComplete.java | 324 ++
.../sample/complete/CompletionCandidate.java | 89 +
.../stream/sample/complete/PojoEvent.java | 46 +
.../sample/complete/StreamingWordExtract.java | 162 +
.../stream/sample/complete/TopNByKey.java | 120 +
.../sample/complete/TopWikipediaSessions.java | 347 ++
.../stream/sample/complete/TrafficRoutes.java | 523 ++
.../sample/complete/TwitterAutoComplete.java | 254 +
.../sample/cookbook/CombinePerKeyExamples.java | 285 ++
.../stream/sample/cookbook/DeDupExample.java | 127 +
.../stream/sample/cookbook/InputPojo.java | 78 +
.../sample/cookbook/MaxPerKeyExamples.java | 205 +
.../stream/sample/cookbook/OutputPojo.java | 56 +
.../stream/sample/cookbook/TriggerExample.java | 577 +++
.../src/main/resources/META-INF/properties.xml | 141 +
.../stream/sample/MinimalWordCountTest.java | 61 +
.../stream/sample/WindowedWordCountTest.java | 91 +
.../sample/complete/AutoCompleteTest.java | 62 +
.../complete/StreamingWordExtractTest.java | 144 +
.../complete/TopWikipediaSessionsTest.java | 74 +
.../sample/complete/TrafficRoutesTest.java | 67 +
.../complete/TwitterAutoCompleteTest.java | 66 +
.../cookbook/CombinePerKeyExamplesTest.java | 56 +
.../sample/cookbook/DeDupExampleTest.java | 59 +
.../sample/cookbook/MaxPerKeyExamplesTest.java | 210 +
.../src/test/resources/data/word.txt | 2 +
.../src/test/resources/log4j.properties | 45 +
.../src/test/resources/sampletweets.txt | 44 +
.../src/test/resources/wordcount/word.txt | 8 +
examples/iteration/pom.xml | 37 +
examples/iteration/src/assemble/appPackage.xml | 59 +
.../apex/examples/iteration/Application.java | 171 +
.../apex/examples/iteration/package-info.java | 22 +
.../src/main/resources/META-INF/properties.xml | 45 +
.../examples/iteration/ApplicationTest.java | 86 +
.../src/test/resources/log4j.properties | 43 +
examples/machinedata/pom.xml | 66 +
.../machinedata/src/assemble/appPackage.xml | 59 +
examples/machinedata/src/main/html/global.js | 269 +
examples/machinedata/src/main/html/index.php | 263 +
examples/machinedata/src/main/html/json.php | 96 +
examples/machinedata/src/main/html/malhar.css | 4688 ++++++++++++++++++
.../apex/examples/machinedata/Application.java | 83 +
.../machinedata/DimensionGenerator.java | 121 +
.../examples/machinedata/InputReceiver.java | 524 ++
.../examples/machinedata/data/AverageData.java | 107 +
.../examples/machinedata/data/MachineInfo.java | 158 +
.../examples/machinedata/data/MachineKey.java | 381 ++
.../examples/machinedata/data/ResourceType.java | 66 +
.../operator/CalculatorOperator.java | 279 ++
.../operator/MachineInfoAveragingOperator.java | 216 +
...chineInfoAveragingPrerequisitesOperator.java | 88 +
.../operator/MachineInfoAveragingUnifier.java | 92 +
.../machinedata/util/Combinatorics.java | 101 +
.../examples/machinedata/util/DataTable.java | 75 +
.../src/main/resources/META-INF/properties.xml | 139 +
.../machinedata/CalculatorOperatorTest.java | 197 +
.../src/test/resources/log4j.properties | 43 +
examples/mobile/pom.xml | 64 +
examples/mobile/src/assemble/appPackage.xml | 59 +
.../apex/examples/mobile/Application.java | 172 +
.../examples/mobile/PhoneEntryOperator.java | 124 +
.../examples/mobile/PhoneMovementGenerator.java | 335 ++
.../apex/examples/mobile/doc-files/Mobile.png | Bin 0 -> 24390 bytes
.../apex/examples/mobile/package-info.java | 22 +
.../src/main/resources/META-INF/properties.xml | 82 +
.../apex/examples/mobile/ApplicationTest.java | 117 +
.../src/test/resources/dt-site-mobile.xml | 87 +
.../mobile/src/test/resources/log4j.properties | 43 +
examples/mrmonitor/pom.xml | 64 +
examples/mrmonitor/src/assemble/appPackage.xml | 59 +
.../apex/examples/mrmonitor/Application.java | 56 +
.../apex/examples/mrmonitor/Constants.java | 48 +
.../examples/mrmonitor/MRJobStatusOperator.java | 622 +++
.../mrmonitor/MRMonitoringApplication.java | 93 +
.../apex/examples/mrmonitor/MRStatusObject.java | 501 ++
.../apache/apex/examples/mrmonitor/MRUtil.java | 99 +
.../mrmonitor/MapToMRObjectOperator.java | 89 +
.../src/main/resources/META-INF/properties.xml | 63 +
.../src/main/resources/mrdebugger.html | 237 +
.../mrmonitor/MrMonitoringApplicationTest.java | 66 +
.../src/test/resources/dt-site-monitoring.xml | 63 +
.../src/test/resources/log4j.properties | 43 +
examples/mroperator/pom.xml | 56 +
examples/mroperator/src/assemble/appPackage.xml | 59 +
.../apex/examples/mroperator/DateWritable.java | 80 +
.../mroperator/HdfsKeyValOutputOperator.java | 41 +
.../mroperator/InvertedIndexApplication.java | 45 +
.../apex/examples/mroperator/LineIndexer.java | 120 +
.../examples/mroperator/LogCountsPerHour.java | 187 +
.../mroperator/LogsCountApplication.java | 46 +
.../apex/examples/mroperator/MapOperator.java | 414 ++
.../mroperator/MapReduceApplication.java | 114 +
.../mroperator/NewWordCountApplication.java | 44 +
.../mroperator/OutputCollectorImpl.java | 94 +
.../examples/mroperator/ReduceOperator.java | 188 +
.../apex/examples/mroperator/ReporterImpl.java | 119 +
.../apex/examples/mroperator/WordCount.java | 104 +
.../src/main/resources/META-INF/properties.xml | 88 +
.../examples/mroperator/MapOperatorTest.java | 159 +
.../examples/mroperator/ReduceOperatorTest.java | 74 +
.../mroperator/WordCountMRApplicationTest.java | 74 +
.../src/test/resources/log4j.properties | 43 +
examples/pi/pom.xml | 45 +
examples/pi/src/assemble/appPackage.xml | 59 +
.../apache/apex/examples/pi/Application.java | 92 +
.../apex/examples/pi/ApplicationAppData.java | 135 +
.../apex/examples/pi/ApplicationWithScript.java | 112 +
.../org/apache/apex/examples/pi/Calculator.java | 81 +
.../apache/apex/examples/pi/NamedValueList.java | 94 +
.../apex/examples/pi/PiCalculateOperator.java | 90 +
.../apex/examples/pi/doc-files/Application.gif | Bin 0 -> 6791 bytes
.../examples/pi/doc-files/ApplicationScript.gif | Bin 0 -> 6687 bytes
.../apache/apex/examples/pi/package-info.java | 22 +
.../src/main/resources/META-INF/properties.xml | 109 +
.../src/main/resources/PiExampleDataSchema.json | 3 +
.../src/main/resources/app/PiJsonExample.json | 52 +
.../apex/examples/pi/ApplicationTest.java | 42 +
.../examples/pi/ApplicationWithScriptTest.java | 35 +
.../apache/apex/examples/pi/CalculatorTest.java | 41 +
examples/pi/src/test/resources/dt-site-pi.xml | 40 +
.../pi/src/test/resources/dt-site-pilibrary.xml | 45 +
examples/pi/src/test/resources/log4j.properties | 43 +
examples/pom.xml | 231 +
examples/r/pom.xml | 83 +
examples/r/src/assemble/appPackage.xml | 59 +
.../examples/r/oldfaithful/FaithfulKey.java | 55 +
.../examples/r/oldfaithful/FaithfulRScript.java | 111 +
.../examples/r/oldfaithful/InputGenerator.java | 145 +
.../r/oldfaithful/OldFaithfulApplication.java | 75 +
.../src/main/resources/META-INF/properties.xml | 41 +
.../apex/examples/r/oldfaithful/eruptionModel.R | 60 +
.../oldfaithful/OldFaithfulApplicationTest.java | 51 +
.../src/test/resources/dt-site-oldfaithful.xml | 41 +
examples/r/src/test/resources/log4j.properties | 43 +
examples/sql/pom.xml | 102 +
examples/sql/src/assemble/appPackage.xml | 59 +
.../sql/sample/FusionStyleSQLApplication.java | 91 +
.../sql/sample/PureStyleSQLApplication.java | 68 +
.../sql/sample/SQLApplicationWithAPI.java | 48 +
.../sql/sample/SQLApplicationWithModelFile.java | 53 +
.../properties-FusionStyleSQLApplication.xml | 65 +
.../properties-PureStyleSQLApplication.xml | 65 +
.../properties-SQLApplicationWithAPI.xml | 43 +
.../properties-SQLApplicationWithModelFile.xml | 32 +
.../src/main/resources/META-INF/properties.xml | 41 +
.../main/resources/model/model_file_csv.json | 27 +
.../sample/FusionStyleSQLApplicationTest.java | 121 +
.../sql/sample/PureStyleSQLApplicationTest.java | 155 +
.../sql/sample/SQLApplicationWithAPITest.java | 92 +
.../sample/SQLApplicationWithModelFileTest.java | 113 +
examples/sql/src/test/resources/input.csv | 6 +
.../sql/src/test/resources/log4j.properties | 50 +
examples/twitter/pom.xml | 101 +
examples/twitter/src/assemble/appPackage.xml | 59 +
.../twitter/KinesisHashtagsApplication.java | 236 +
.../apex/examples/twitter/SlidingContainer.java | 70 +
.../twitter/TwitterDumpApplication.java | 117 +
.../twitter/TwitterDumpHBaseApplication.java | 81 +
.../twitter/TwitterStatusHashtagExtractor.java | 52 +
.../twitter/TwitterStatusURLExtractor.java | 56 +
.../twitter/TwitterStatusWordExtractor.java | 62 +
.../twitter/TwitterTopCounterApplication.java | 222 +
.../twitter/TwitterTopWordsApplication.java | 71 +
.../TwitterTrendingHashtagsApplication.java | 166 +
.../apache/apex/examples/twitter/URLSerDe.java | 74 +
.../examples/twitter/WindowedTopCounter.java | 282 ++
.../examples/twitter/doc-files/Application.gif | Bin 0 -> 8775 bytes
.../apex/examples/twitter/package-info.java | 22 +
.../META-INF/properties-TwitterKinesisDemo.xml | 52 +
.../src/main/resources/META-INF/properties.xml | 121 +
examples/twitter/src/main/resources/mysql.sql | 35 +
.../src/main/resources/top_urls.tplg.properties | 48 +
.../resources/twitterHashTagDataSchema.json | 4 +
.../main/resources/twitterURLDataSchema.json | 4 +
.../main/resources/twitterWordDataSchema.json | 4 +
.../twitter/TwitterDumpApplicationTest.java | 48 +
.../examples/twitter/TwitterTopCounterTest.java | 45 +
.../examples/twitter/TwitterTopWordsTest.java | 48 +
.../test/resources/dt-site-rollingtopwords.xml | 73 +
.../twitter/src/test/resources/log4j.properties | 43 +
examples/uniquecount/pom.xml | 50 +
.../uniquecount/src/assemble/appPackage.xml | 59 +
.../apex/examples/uniquecount/Application.java | 90 +
.../examples/uniquecount/CountVerifier.java | 107 +
.../uniquecount/RandomDataGenerator.java | 115 +
.../examples/uniquecount/RandomKeyValues.java | 153 +
.../uniquecount/RandomKeysGenerator.java | 153 +
.../uniquecount/UniqueKeyValCountExample.java | 64 +
.../apex/examples/uniquecount/package-info.java | 22 +
.../src/main/resources/META-INF/properties.xml | 29 +
.../examples/uniquecount/ApplicationTest.java | 38 +
.../uniquecount/UniqueKeyValExampleTest.java | 38 +
.../src/test/resources/log4j.properties | 43 +
examples/wordcount/pom.xml | 49 +
examples/wordcount/src/assemble/appPackage.xml | 59 +
.../apex/examples/wordcount/Application.java | 108 +
.../wordcount/ApplicationWithQuerySupport.java | 133 +
.../apex/examples/wordcount/FileWordCount.java | 349 ++
.../apex/examples/wordcount/LineReader.java | 118 +
.../apache/apex/examples/wordcount/WCPair.java | 62 +
.../examples/wordcount/WindowWordCount.java | 94 +
.../wordcount/WordCountInputOperator.java | 109 +
.../examples/wordcount/WordCountWriter.java | 109 +
.../apex/examples/wordcount/WordReader.java | 98 +
.../wordcount/doc-files/UniqueWordCounter.jpg | Bin 0 -> 65015 bytes
.../apex/examples/wordcount/package-info.java | 22 +
.../src/main/resources/META-INF/properties.xml | 98 +
.../src/main/resources/WordDataSchema.json | 4 +
.../apex/examples/wordcount/samplefile.txt | 1 +
.../wordcount/src/main/resources/samplefile.txt | 2 +
.../examples/wordcount/ApplicationTest.java | 51 +
.../src/test/resources/dt-site-wordcount.xml | 37 +
.../src/test/resources/log4j.properties | 43 +
examples/yahoofinance/pom.xml | 65 +
.../yahoofinance/src/assemble/appPackage.xml | 59 +
.../yahoofinance/ApplicationWithDerbySQL.java | 80 +
.../examples/yahoofinance/StockTickInput.java | 192 +
.../yahoofinance/YahooFinanceApplication.java | 365 ++
.../YahooFinanceCSVInputOperator.java | 160 +
.../yahoofinance/doc-files/Application.gif | Bin 0 -> 12457 bytes
.../examples/yahoofinance/package-info.java | 22 +
.../src/main/resources/META-INF/properties.xml | 81 +
.../examples/yahoofinance/ApplicationTest.java | 46 +
.../ApplicationWithDerbySQLTest.java | 50 +
.../src/test/resources/alert_create.json | 24 +
.../src/test/resources/log4j.properties | 43 +
.../test/resources/SocketInputOperatorTest.txt | 2 +-
pom.xml | 2 +-
581 files changed, 33391 insertions(+), 33626 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 4781031..4016151 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,7 @@
Apache Apex Malhar
===============================
-Malhar repository contains open source operator and codec library that can be used with the Apache Apex platform to build Realtime streaming applications. In addition to the library there are benchmark, contrib, demos, webdemos and samples folders available. Demos contain demo applications built using the library operators. Webdemos contain webpages for the demos. Benchmark contains performance testing applications. Contrib contains additional operators that interface with third party softwares. Samples contain some sample code that shows how to use the library operators.
+Malhar repository contains open source operator and codec library that can be used with the Apache Apex platform to build Realtime streaming applications. In addition to the library there are benchmark, contrib, examples and samples folders available. Examples contain applications built using the library operators. Benchmark contains performance testing applications. Contrib contains additional operators that interface with third party softwares. Samples contain some sample code that shows how to use the library operators.
Contributing
------------
@@ -32,7 +32,7 @@ mvn install -DskipTests
The code depends on the Apex API, which is available in the Maven Central Repository. Individual modules such as library can be built independently by changing directory to the module and running maven there.
-Multiple [demo applications](demos/) are provided to showcase the Apex platform and application development process.
+Multiple [example applications](examples/) are provided to showcase the Apex platform and application development process.
License
--------------------
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/distributedistinct/pom.xml
----------------------------------------------------------------------
diff --git a/demos/distributedistinct/pom.xml b/demos/distributedistinct/pom.xml
deleted file mode 100644
index 3bb4400..0000000
--- a/demos/distributedistinct/pom.xml
+++ /dev/null
@@ -1,41 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>distributedistinct</artifactId>
- <packaging>jar</packaging>
-
- <name>Apache Apex Malhar Distributed Distinct Demo</name>
- <description></description>
-
- <parent>
- <groupId>org.apache.apex</groupId>
- <artifactId>malhar-demos</artifactId>
- <version>3.7.0-SNAPSHOT</version>
- </parent>
-
- <properties>
- <skipTests>true</skipTests>
- </properties>
-
-</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/distributedistinct/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/demos/distributedistinct/src/assemble/appPackage.xml b/demos/distributedistinct/src/assemble/appPackage.xml
deleted file mode 100644
index 4138cf2..0000000
--- a/demos/distributedistinct/src/assemble/appPackage.xml
+++ /dev/null
@@ -1,59 +0,0 @@
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
- <id>appPackage</id>
- <formats>
- <format>jar</format>
- </formats>
- <includeBaseDirectory>false</includeBaseDirectory>
- <fileSets>
- <fileSet>
- <directory>${basedir}/target/</directory>
- <outputDirectory>/app</outputDirectory>
- <includes>
- <include>${project.artifactId}-${project.version}.jar</include>
- </includes>
- </fileSet>
- <fileSet>
- <directory>${basedir}/target/deps</directory>
- <outputDirectory>/lib</outputDirectory>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/site/conf</directory>
- <outputDirectory>/conf</outputDirectory>
- <includes>
- <include>*.xml</include>
- </includes>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/main/resources/META-INF</directory>
- <outputDirectory>/META-INF</outputDirectory>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/main/resources/app</directory>
- <outputDirectory>/app</outputDirectory>
- </fileSet>
- </fileSets>
-
-</assembly>
-
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/distributedistinct/src/main/java/com/datatorrent/demos/distributeddistinct/Application.java
----------------------------------------------------------------------
diff --git a/demos/distributedistinct/src/main/java/com/datatorrent/demos/distributeddistinct/Application.java b/demos/distributedistinct/src/main/java/com/datatorrent/demos/distributeddistinct/Application.java
deleted file mode 100644
index 656b083..0000000
--- a/demos/distributedistinct/src/main/java/com/datatorrent/demos/distributeddistinct/Application.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.distributeddistinct;
-
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-import com.datatorrent.lib.algo.UniqueValueCount;
-import com.datatorrent.lib.io.ConsoleOutputOperator;
-import com.datatorrent.lib.stream.Counter;
-import com.datatorrent.lib.stream.StreamDuplicater;
-import com.datatorrent.lib.util.KeyValPair;
-
-/**
- * This application demonstrates the UniqueValueCount operator. It uses an input operator which generates random key
- * value pairs and simultaneously emits them to the UniqueValueCount operator and keeps track of the number of unique
- * values per key to emit to the verifier.
- *
- * @since 1.0.4
- */
-@ApplicationAnnotation(name = "ValueCount")
-public class Application implements StreamingApplication
-{
-
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- RandomKeyValGenerator randGen = dag.addOperator("RandomGenerator", new RandomKeyValGenerator());
- UniqueValueCount<Integer> valCount = dag.addOperator("UniqueCounter", new UniqueValueCount<Integer>());
- ConsoleOutputOperator consOut = dag.addOperator("Console", new ConsoleOutputOperator());
- StreamDuplicater<KeyValPair<Integer, Integer>> dup = dag.addOperator("Duplicator", new StreamDuplicater<KeyValPair<Integer, Integer>>());
- CountVerifier verifier = dag.addOperator("Verifier", new CountVerifier());
- ConsoleOutputOperator successOutput = dag.addOperator("Success", new ConsoleOutputOperator());
- successOutput.setStringFormat("Success %d");
- ConsoleOutputOperator failureOutput = dag.addOperator("Failure", new ConsoleOutputOperator());
- failureOutput.setStringFormat("Failure %d");
-
- Counter successcounter = dag.addOperator("SuccessCounter", new Counter());
- Counter failurecounter = dag.addOperator("FailureCounter", new Counter());
-
- dag.addStream("Events", randGen.outport, valCount.input);
- dag.addStream("Duplicates", valCount.output, dup.data);
- dag.addStream("Unverified", dup.out1, verifier.recIn);
- dag.addStream("EventCount", randGen.verport, verifier.trueIn);
- dag.addStream("Verified", verifier.successPort, successcounter.input);
- dag.addStream("Failed", verifier.failurePort, failurecounter.input);
- dag.addStream("SuccessCount", successcounter.output, successOutput.input);
- dag.addStream("FailedCount", failurecounter.output, failureOutput.input);
- dag.addStream("Output", dup.out2, consOut.input);
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/distributedistinct/src/main/java/com/datatorrent/demos/distributeddistinct/CountVerifier.java
----------------------------------------------------------------------
diff --git a/demos/distributedistinct/src/main/java/com/datatorrent/demos/distributeddistinct/CountVerifier.java b/demos/distributedistinct/src/main/java/com/datatorrent/demos/distributeddistinct/CountVerifier.java
deleted file mode 100644
index 417ed7c..0000000
--- a/demos/distributedistinct/src/main/java/com/datatorrent/demos/distributeddistinct/CountVerifier.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.distributeddistinct;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import com.datatorrent.api.Context.OperatorContext;
-import com.datatorrent.api.DefaultInputPort;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.api.Operator;
-import com.datatorrent.api.annotation.OutputPortFieldAnnotation;
-import com.datatorrent.lib.util.KeyValPair;
-
-/**
- * An operator that takes two streams of Integer to Integer KeyValPairs and verifies that the two streams output the
- * same KeyValPairs within a given window.
- *
- * @since 1.0.4
- */
-public class CountVerifier implements Operator
-{
- Map<Integer, Integer> trueCount = new HashMap<Integer, Integer>();
- Map<Integer, Integer> receivedCount = new HashMap<Integer, Integer>();
-
- public final transient DefaultInputPort<KeyValPair<Integer, Integer>> trueIn = new DefaultInputPort<KeyValPair<Integer, Integer>>()
- {
- @Override
- public void process(KeyValPair<Integer, Integer> tuple)
- {
- trueCount.put(tuple.getKey(), tuple.getValue());
- }
- };
-
- public final transient DefaultInputPort<KeyValPair<Integer, Integer>> recIn = new DefaultInputPort<KeyValPair<Integer, Integer>>()
- {
- @Override
- public void process(KeyValPair<Integer, Integer> tuple)
- {
- receivedCount.put(tuple.getKey(), tuple.getValue());
- }
- };
-
- @OutputPortFieldAnnotation(optional = true)
- public final transient DefaultOutputPort<Integer> successPort = new DefaultOutputPort<Integer>();
- @OutputPortFieldAnnotation(optional = true)
- public final transient DefaultOutputPort<Integer> failurePort = new DefaultOutputPort<Integer>();
-
- @Override
- public void setup(OperatorContext arg0)
- {
-
- }
-
- @Override
- public void teardown()
- {
-
- }
-
- @Override
- public void beginWindow(long windowID)
- {
- }
-
- /**
- * Checks that the key to value pairs are the same and counts the number of pairs that are different. If there are
- * failures, it will emit the number of failures to the failure port. Otherwise, it will emit the number of keys to
- * the success port.
- */
- @Override
- public void endWindow()
- {
- int failureCount = 0;
- for (Map.Entry<Integer, Integer> e : receivedCount.entrySet()) {
- Integer key = e.getKey();
- if (!trueCount.get(key).equals(e.getValue())) {
- failureCount++;
- }
- }
- if (failureCount != 0) {
- failurePort.emit(failureCount);
- } else {
- successPort.emit(trueCount.size());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/distributedistinct/src/main/java/com/datatorrent/demos/distributeddistinct/IntegerUniqueValueCountAppender.java
----------------------------------------------------------------------
diff --git a/demos/distributedistinct/src/main/java/com/datatorrent/demos/distributeddistinct/IntegerUniqueValueCountAppender.java b/demos/distributedistinct/src/main/java/com/datatorrent/demos/distributeddistinct/IntegerUniqueValueCountAppender.java
deleted file mode 100644
index bf8a307..0000000
--- a/demos/distributedistinct/src/main/java/com/datatorrent/demos/distributeddistinct/IntegerUniqueValueCountAppender.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.distributeddistinct;
-
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.HashSet;
-import java.util.Set;
-
-import com.datatorrent.lib.algo.UniqueValueCount;
-import com.datatorrent.lib.util.KeyValPair;
-
-/**
- * This operator demonstrates {@link UniqueValueCountAppender} given that the keys and values of the preceding {@link UniqueValueCount} operator
- * are both integers. <br/>
- * It will keep track of the number of all the unique values emitted per key since the application starts.
- *
- * @since 1.0.4
- */
-public class IntegerUniqueValueCountAppender extends UniqueValueCountAppender<Integer>
-{
- @Override
- public Object processResultSet(ResultSet resultSet)
- {
- Set<Integer> valSet = new HashSet<Integer>();
- try {
- while (resultSet.next()) {
- valSet.add(resultSet.getInt(1));
- }
- return valSet;
- } catch (SQLException e) {
- throw new RuntimeException("while processing the result set", e);
- }
- }
-
- @Override
- protected void prepareGetStatement(PreparedStatement getStatement, Object key) throws SQLException
- {
- getStatement.setInt(1, (Integer)key);
- }
-
- @Override
- protected void preparePutStatement(PreparedStatement putStatement, Object key, Object value) throws SQLException
- {
- @SuppressWarnings("unchecked")
- Set<Integer> valueSet = (Set<Integer>)value;
- for (Integer val : valueSet) {
- @SuppressWarnings("unchecked")
- Set<Integer> currentVals = (Set<Integer>)get(key);
- if (!currentVals.contains(val)) {
- batch = true;
- putStatement.setInt(1, (Integer)key);
- putStatement.setInt(2, val);
- putStatement.setLong(3, windowID);
- putStatement.addBatch();
- }
- }
- }
-
- @Override
- public void endWindow()
- {
- try {
- Statement stmt = store.getConnection().createStatement();
- String keySetQuery = "SELECT DISTINCT col1 FROM " + tableName;
- ResultSet resultSet = stmt.executeQuery(keySetQuery);
- while (resultSet.next()) {
- int val = resultSet.getInt(1);
- @SuppressWarnings("unchecked")
- Set<Integer> valSet = (Set<Integer>)cacheManager.get(val);
- output.emit(new KeyValPair<Object, Object>(val, valSet.size()));
- }
- } catch (SQLException e) {
- throw new RuntimeException("While emitting tuples", e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/distributedistinct/src/main/java/com/datatorrent/demos/distributeddistinct/RandomKeyValGenerator.java
----------------------------------------------------------------------
diff --git a/demos/distributedistinct/src/main/java/com/datatorrent/demos/distributeddistinct/RandomKeyValGenerator.java b/demos/distributedistinct/src/main/java/com/datatorrent/demos/distributeddistinct/RandomKeyValGenerator.java
deleted file mode 100644
index c8016da..0000000
--- a/demos/distributedistinct/src/main/java/com/datatorrent/demos/distributeddistinct/RandomKeyValGenerator.java
+++ /dev/null
@@ -1,186 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.distributeddistinct;
-
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Random;
-import java.util.Set;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.datatorrent.api.Context.OperatorContext;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.api.InputOperator;
-import com.datatorrent.api.annotation.OutputPortFieldAnnotation;
-import com.datatorrent.lib.util.KeyHashValPair;
-import com.datatorrent.lib.util.KeyValPair;
-
-/**
- * Generates random KeyValPairs and optionally, keeps track of the number of unique values per pair to emit to the
- * verification port.
- *
- * @since 1.0.4
- */
-public class RandomKeyValGenerator implements InputOperator
-{
-
- protected int numKeys = 50;
- protected int numVals = 1000;
- protected int tupleBlast = 1000;
- protected Map<Integer, Set<Integer>> valhistory = new HashMap<Integer, Set<Integer>>();
- private Random rand = new Random();
- private boolean once;
- private boolean clearHistory;
- @OutputPortFieldAnnotation(optional = false)
- public transient DefaultOutputPort<KeyValPair<Integer, Object>> outport = new DefaultOutputPort<KeyValPair<Integer, Object>>();
-
- @OutputPortFieldAnnotation(optional = true)
- public transient DefaultOutputPort<KeyValPair<Integer, Integer>> verport = new DefaultOutputPort<KeyValPair<Integer, Integer>>();
-
- /**
- * Ensures that the generator emits KeyValPairs once per window
- */
- @Override
- public void beginWindow(long l)
- {
- once = false;
- }
-
- /**
- * Emits the total count of unique values per key as KeyHashValPairs to the verification port
- */
- @Override
- public void endWindow()
- {
- if (verport.isConnected()) {
- for (Map.Entry<Integer, Set<Integer>> e : valhistory.entrySet()) {
- verport.emit(new KeyHashValPair<Integer, Integer>(e.getKey(), e.getValue().size()));
- }
- }
- if (clearHistory) {
- valhistory.clear();
- }
- }
-
- @Override
- public void setup(OperatorContext arg0)
- {
-
- }
-
- @Override
- public void teardown()
- {
-
- }
-
- /**
- * Emits random KeyValPairs and keeps track of the unique values per key.
- */
- @Override
- public void emitTuples()
- {
- if (!once) {
- int key;
- int val;
- for (int i = 0; i < tupleBlast; i++) {
- key = rand.nextInt(numKeys);
- val = rand.nextInt(numVals);
- outport.emit(new KeyValPair<Integer, Object>(key, val));
- if (verport.isConnected()) {
- Set<Integer> count = valhistory.get(key);
- if (count == null) {
- Set<Integer> tempset = new HashSet<Integer>();
- tempset.add(val);
- valhistory.put(key, tempset);
- LOG.debug("key {} val {}", key, tempset);
- } else if (!valhistory.get(key).contains(val)) {
- valhistory.get(key).add(val);
- }
- }
- }
- once = true;
- }
- }
-
- /**
- * @return the number of possible keys
- */
- public int getNumKeys()
- {
- return numKeys;
- }
-
- /**
- * Sets the number of possible keys to numKeys
- *
- * @param numKeys
- * the new number of possible keys
- */
- public void setNumKeys(int numKeys)
- {
- this.numKeys = numKeys;
- }
-
- /**
- * Returns the number of possible values that can be emitted
- *
- * @return the number of possible values that can be emitted
- */
- public int getNumVals()
- {
- return numVals;
- }
-
- /**
- * Sets the number of possible values that can be emitted to numVals
- *
- * @param numVals
- * the number of possible values that can be emitted
- */
- public void setNumVals(int numVals)
- {
- this.numVals = numVals;
- }
-
- /**
- * Sets the number of KeyValPairs to be emitted to tupleBlast
- *
- * @param tupleBlast
- * the new number of KeyValPairs to be emitted
- */
- public void setTupleBlast(int tupleBlast)
- {
- this.tupleBlast = tupleBlast;
- }
-
- /**
- * tuple blast
- * @return
- */
- public int getTupleBlast()
- {
- return tupleBlast;
- }
-
- private static final Logger LOG = LoggerFactory.getLogger(RandomKeyValGenerator.class);
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/distributedistinct/src/main/java/com/datatorrent/demos/distributeddistinct/StatefulApplication.java
----------------------------------------------------------------------
diff --git a/demos/distributedistinct/src/main/java/com/datatorrent/demos/distributeddistinct/StatefulApplication.java b/demos/distributedistinct/src/main/java/com/datatorrent/demos/distributeddistinct/StatefulApplication.java
deleted file mode 100644
index d6c488c..0000000
--- a/demos/distributedistinct/src/main/java/com/datatorrent/demos/distributeddistinct/StatefulApplication.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.distributeddistinct;
-
-import java.io.Serializable;
-
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.Context;
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-import com.datatorrent.lib.algo.UniqueValueCount;
-import com.datatorrent.lib.algo.UniqueValueCount.InternalCountOutput;
-import com.datatorrent.lib.codec.KryoSerializableStreamCodec;
-import com.datatorrent.lib.io.ConsoleOutputOperator;
-
-/**
- * This application demonstrates the UniqueValueCount operator. It uses an input operator which generates random key
- * value pairs and emits them to the UniqueValueCount operator which keeps track of the unique values per window. It
- * then emits the values to the StatefulUniqueCount which uses a combination of a cache and database to keep track of
- * the overall unique values and outputs the resulting unique value count to the ConsoleOutputOperator.
- *
- * @since 1.0.4
- */
-@ApplicationAnnotation(name = "StatefulDistinctCount")
-public class StatefulApplication implements StreamingApplication
-{
- @SuppressWarnings("unchecked")
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- RandomKeyValGenerator randGen = dag.addOperator("RandomGenerator", new RandomKeyValGenerator());
- UniqueValueCount<Integer> valCount = dag.addOperator("UniqueCounter", new UniqueValueCount<Integer>());
- ConsoleOutputOperator consOut = dag.addOperator("Console", new ConsoleOutputOperator());
- IntegerUniqueValueCountAppender uniqueUnifier = dag.addOperator("StatefulUniqueCounter", new IntegerUniqueValueCountAppender());
- dag.getOperatorMeta("StatefulUniqueCounter").getMeta(uniqueUnifier.input).getAttributes().put(Context.PortContext.STREAM_CODEC, new KeyBasedStreamCodec());
-
- @SuppressWarnings("rawtypes")
- DefaultOutputPort valOut = valCount.output;
- @SuppressWarnings("rawtypes")
- DefaultOutputPort uniqueOut = uniqueUnifier.output;
-
- dag.addStream("Events", randGen.outport, valCount.input);
- dag.addStream("Unified", valOut, uniqueUnifier.input);
- dag.addStream("Result", uniqueOut, consOut.input);
- }
-
- public static class KeyBasedStreamCodec extends KryoSerializableStreamCodec<InternalCountOutput<Integer>> implements Serializable
- {
- @Override
- public int getPartition(InternalCountOutput<Integer> t)
- {
- return t.getKey().hashCode();
- }
-
- private static final long serialVersionUID = 201407231527L;
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/distributedistinct/src/main/java/com/datatorrent/demos/distributeddistinct/UniqueValueCountAppender.java
----------------------------------------------------------------------
diff --git a/demos/distributedistinct/src/main/java/com/datatorrent/demos/distributeddistinct/UniqueValueCountAppender.java b/demos/distributedistinct/src/main/java/com/datatorrent/demos/distributeddistinct/UniqueValueCountAppender.java
deleted file mode 100644
index 3f14e0f..0000000
--- a/demos/distributedistinct/src/main/java/com/datatorrent/demos/distributeddistinct/UniqueValueCountAppender.java
+++ /dev/null
@@ -1,236 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.distributeddistinct;
-
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Map;
-import java.util.Set;
-
-import javax.annotation.Nonnull;
-import javax.validation.constraints.Min;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-
-import com.datatorrent.api.Context;
-import com.datatorrent.api.DefaultPartition;
-import com.datatorrent.api.Partitioner;
-import com.datatorrent.lib.algo.UniqueValueCount;
-import com.datatorrent.lib.algo.UniqueValueCount.InternalCountOutput;
-import com.datatorrent.lib.db.jdbc.JDBCLookupCacheBackedOperator;
-import com.datatorrent.netlet.util.DTThrowable;
-
-/**
- * <p>
- * This operator supplements the {@link UniqueValueCount} operator by making it state-full.<br/>
- * It helps to track unique values through out the lifetime of the application.
- * </p>
- *
- * <p>
- * The operator counts the number of values emitted per key by storing previously
- * counted values in both a transient cache and in a persistent database. <br/>
- * In case of a rollback, it will erase all values committed to the database
- * in the windows greater than the activation window, then re-enter them as needed to keep it stateful.<br/>
- * This operator, when appended to {@link UniqueValueCount} will keep track of the
- * unique values emitted since the start of the application.
- *
- * @since 1.0.4
- */
-public abstract class UniqueValueCountAppender<V> extends JDBCLookupCacheBackedOperator<InternalCountOutput<V>> implements Partitioner<UniqueValueCountAppender<V>>
-{
- protected Set<Integer> partitionKeys;
- protected int partitionMask;
- protected transient long windowID;
- protected transient boolean batch;
- @Min(1)
- private int partitionCount = 1;
-
- public UniqueValueCountAppender()
-
- {
- partitionKeys = Sets.newHashSet(0);
- partitionMask = 0;
- }
-
- public void setPartitionCount(int partitionCount)
- {
- this.partitionCount = partitionCount;
- }
-
- public int getPartitionCount()
- {
- return partitionCount;
- }
-
- @Override
- public void setup(Context.OperatorContext context)
- {
- super.setup(context);
- LOGGER.debug("store properties {} {}", store.getDatabaseDriver(), store.getDatabaseUrl());
- LOGGER.debug("table name {}", tableName);
- windowID = context.getValue(Context.OperatorContext.ACTIVATION_WINDOW_ID);
- try {
- ResultSet resultSet = store.getConnection().createStatement().executeQuery("SELECT col1 FROM " + tableName + " WHERE col3 >= " + windowID);
- PreparedStatement deleteStatement = store.getConnection().prepareStatement("DELETE FROM " + tableName + " WHERE col3 >= " + windowID + " AND col1 = ?");
-
- Set<Object> deletedKeys = Sets.newHashSet();
- while (resultSet.next()) {
- Object key = resultSet.getObject(1);
- if (partitionKeys.contains((key.hashCode() & partitionMask)) && !deletedKeys.contains(key)) {
- deletedKeys.add(key);
- deleteStatement.setObject(1, key);
- deleteStatement.executeUpdate();
- }
- }
- } catch (SQLException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- protected void processTuple(InternalCountOutput<V> tuple)
- {
-
- Object key = getKeyFromTuple(tuple);
- @SuppressWarnings("unchecked")
- Set<Object> values = (Set<Object>)cacheManager.get(key);
- if (values == null) {
- values = Sets.newHashSet();
- }
- values.addAll(tuple.getInternalSet());
- cacheManager.put(key, values);
- }
-
- @Override
- protected String fetchInsertQuery()
- {
- return "INSERT INTO " + tableName + " (col1, col2, col3) VALUES (?, ?, ?)";
- }
-
- @Override
- protected String fetchGetQuery()
- {
- return "select col2 from " + tableName + " where col1 = ?";
- }
-
- @Override
- public Map<Object, Object> loadInitialData()
- {
- return null;
- }
-
- @Override
- public void put(@Nonnull Object key, @Nonnull Object value)
- {
- try {
- batch = false;
- preparePutStatement(putStatement, key, value);
- if (batch) {
- putStatement.executeBatch();
- putStatement.clearBatch();
- }
- } catch (SQLException e) {
- throw new RuntimeException("while executing insert", e);
- }
- }
-
- @Override
- public void teardown()
- {
-
- }
-
- @Override
- public void beginWindow(long windowID)
- {
- this.windowID = windowID;
- }
-
- @Override
- protected Object getKeyFromTuple(InternalCountOutput<V> tuple)
- {
- return tuple.getKey();
- }
-
- @Override
- public void putAll(Map<Object, Object> m)
- {
- throw new UnsupportedOperationException("not supported");
- }
-
- @Override
- public void remove(Object key)
- {
- throw new UnsupportedOperationException("not supported");
- }
-
- /**
- * Assigns the partitions according to certain key values and keeps track of the
- * keys that each partition will be processing so that in the case of a
- * rollback, each partition will only clear the data that it is responsible for.
- */
- @Override
- public Collection<com.datatorrent.api.Partitioner.Partition<UniqueValueCountAppender<V>>> definePartitions(Collection<com.datatorrent.api.Partitioner.Partition<UniqueValueCountAppender<V>>> partitions, PartitioningContext context)
- {
- final int finalCapacity = DefaultPartition.getRequiredPartitionCount(context, this.partitionCount);
- UniqueValueCountAppender<V> anOldOperator = partitions.iterator().next().getPartitionedInstance();
- partitions.clear();
-
- Collection<Partition<UniqueValueCountAppender<V>>> newPartitions = Lists.newArrayListWithCapacity(finalCapacity);
-
- for (int i = 0; i < finalCapacity; i++) {
- try {
- @SuppressWarnings("unchecked")
- UniqueValueCountAppender<V> statefulUniqueCount = this.getClass().newInstance();
- DefaultPartition<UniqueValueCountAppender<V>> partition = new DefaultPartition<UniqueValueCountAppender<V>>(statefulUniqueCount);
- newPartitions.add(partition);
- } catch (Throwable cause) {
- DTThrowable.rethrow(cause);
- }
- }
-
- DefaultPartition.assignPartitionKeys(Collections.unmodifiableCollection(newPartitions), input);
- int lPartitionMask = newPartitions.iterator().next().getPartitionKeys().get(input).mask;
-
- for (Partition<UniqueValueCountAppender<V>> statefulUniqueCountPartition : newPartitions) {
- UniqueValueCountAppender<V> statefulUniqueCountInstance = statefulUniqueCountPartition.getPartitionedInstance();
-
- statefulUniqueCountInstance.partitionKeys = statefulUniqueCountPartition.getPartitionKeys().get(input).partitions;
- statefulUniqueCountInstance.partitionMask = lPartitionMask;
- statefulUniqueCountInstance.store = anOldOperator.store;
- statefulUniqueCountInstance.tableName = anOldOperator.tableName;
- statefulUniqueCountInstance.cacheManager = anOldOperator.cacheManager;
- }
- return newPartitions;
- }
-
- @Override
- public void partitioned(Map<Integer, com.datatorrent.api.Partitioner.Partition<UniqueValueCountAppender<V>>> partitions)
- {
- }
-
- private static final Logger LOGGER = LoggerFactory.getLogger(UniqueValueCountAppender.class);
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/distributedistinct/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/demos/distributedistinct/src/main/resources/META-INF/properties.xml b/demos/distributedistinct/src/main/resources/META-INF/properties.xml
deleted file mode 100644
index 8742328..0000000
--- a/demos/distributedistinct/src/main/resources/META-INF/properties.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-<?xml version="1.0"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<configuration>
- <!--
- <property>
- <name>dt.application.{appName}.operator.{opName}.prop.{propName}</name>
- <value>some-default-value (if value is not specified, it is required from the user or custom config when launching)</value>
- </property>
- -->
-</configuration>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/distributedistinct/src/site/conf/my-app-conf1.xml
----------------------------------------------------------------------
diff --git a/demos/distributedistinct/src/site/conf/my-app-conf1.xml b/demos/distributedistinct/src/site/conf/my-app-conf1.xml
deleted file mode 100644
index f35873b..0000000
--- a/demos/distributedistinct/src/site/conf/my-app-conf1.xml
+++ /dev/null
@@ -1,27 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<configuration>
- <property>
- <name>dt.attr.MASTER_MEMORY_MB</name>
- <value>1024</value>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/distributedistinct/src/test/java/com/datatorrent/demos/distributeddistinct/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/demos/distributedistinct/src/test/java/com/datatorrent/demos/distributeddistinct/ApplicationTest.java b/demos/distributedistinct/src/test/java/com/datatorrent/demos/distributeddistinct/ApplicationTest.java
deleted file mode 100644
index ef5473f..0000000
--- a/demos/distributedistinct/src/test/java/com/datatorrent/demos/distributeddistinct/ApplicationTest.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.distributeddistinct;
-
-import org.junit.Test;
-
-import com.datatorrent.api.LocalMode;
-
-public class ApplicationTest
-{
- @Test
- public void testSomeMethod() throws Exception
- {
- LocalMode.runApp(new Application(), 15000);
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/distributedistinct/src/test/java/com/datatorrent/demos/distributeddistinct/DistributedDistinctTest.java
----------------------------------------------------------------------
diff --git a/demos/distributedistinct/src/test/java/com/datatorrent/demos/distributeddistinct/DistributedDistinctTest.java b/demos/distributedistinct/src/test/java/com/datatorrent/demos/distributeddistinct/DistributedDistinctTest.java
deleted file mode 100644
index e013217..0000000
--- a/demos/distributedistinct/src/test/java/com/datatorrent/demos/distributeddistinct/DistributedDistinctTest.java
+++ /dev/null
@@ -1,200 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.distributeddistinct;
-
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Properties;
-import java.util.Set;
-
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.datatorrent.api.Context.OperatorContext;
-import com.datatorrent.api.DAG;
-import com.datatorrent.lib.algo.UniqueValueCount.InternalCountOutput;
-import com.datatorrent.lib.helper.OperatorContextTestHelper;
-import com.datatorrent.lib.helper.OperatorContextTestHelper.TestIdOperatorContext;
-
-/**
- * Test for {@link IntegerUniqueValueCountAppender} and {@link UniqueValueCountAppender}
- */
-public class DistributedDistinctTest
-{
- private static final Logger logger = LoggerFactory.getLogger(DistributedDistinctTest.class);
-
- private static final String APP_ID = "DistributedDistinctTest";
- private static final int OPERATOR_ID = 0;
-
- public static final String INMEM_DB_URL = "jdbc:hsqldb:mem:test;sql.syntax_mys=true";
- public static final String INMEM_DB_DRIVER = "org.hsqldb.jdbc.JDBCDriver";
- public static final String TABLE_NAME = "Test_Lookup_Cache";
-
- private static IntegerUniqueValueCountAppender valueCounter;
- private static String applicationPath;
-
- @Test
- public void testProcess() throws Exception
- {
- insertValues();
- Statement stmt = valueCounter.getStore().getConnection().createStatement();
-
- ResultSet resultSet = stmt.executeQuery("SELECT col2 FROM " + TABLE_NAME + " WHERE col1 = 1");
- ArrayList<Integer> answersOne = new ArrayList<Integer>();
- for (int i = 1; i < 16; i++) {
- answersOne.add(i);
- }
- Assert.assertEquals(answersOne, processResult(resultSet));
-
- resultSet = stmt.executeQuery("SELECT col2 FROM " + TABLE_NAME + " WHERE col1 = 2");
- ArrayList<Integer> answersTwo = new ArrayList<Integer>();
- answersTwo.add(3);
- answersTwo.add(6);
- answersTwo.add(9);
- for (int i = 11; i < 21; i++) {
- answersTwo.add(i);
- }
- Assert.assertEquals(answersTwo, processResult(resultSet));
-
- resultSet = stmt.executeQuery("SELECT col2 FROM " + TABLE_NAME + " WHERE col1 = 3");
- ArrayList<Integer> answersThree = new ArrayList<Integer>();
- answersThree.add(2);
- answersThree.add(4);
- answersThree.add(6);
- answersThree.add(8);
- answersThree.add(10);
- for (int i = 11; i < 21; i++) {
- answersThree.add(i);
- }
- Assert.assertEquals(answersThree, processResult(resultSet));
-
- valueCounter.teardown();
- }
-
- public static void insertValues()
- {
- logger.debug("start round 0");
- valueCounter.beginWindow(0);
- emitKeyVals(1, 1, 10, 1);
- emitKeyVals(1, 5, 15, 1);
- valueCounter.endWindow();
- logger.debug("end round 0");
-
- logger.debug("start round 1");
- valueCounter.beginWindow(1);
- emitKeyVals(2, 3, 15, 3);
- emitKeyVals(3, 2, 20, 2);
- emitKeyVals(3, 11, 20, 1);
- valueCounter.endWindow();
- logger.debug("end round 1");
-
- logger.debug("start round 2");
- valueCounter.beginWindow(2);
- emitKeyVals(3, 2, 20, 2);
- emitKeyVals(2, 11, 20, 1);
- valueCounter.endWindow();
- logger.debug("end round 2");
- }
-
- public static ArrayList<Integer> processResult(ResultSet resultSet)
- {
- ArrayList<Integer> tempList = new ArrayList<Integer>();
- try {
- while (resultSet.next()) {
- tempList.add(resultSet.getInt(1));
- }
- } catch (SQLException e) {
- throw new RuntimeException(e);
- }
- Collections.sort(tempList);
- return tempList;
- }
-
- public static void emitKeyVals(int key, int start, int end, int increment)
- {
- int count = 0;
- Set<Object> valSet = new HashSet<Object>();
- for (int i = start; i <= end; i += increment) {
- count++;
- valSet.add(i);
- }
- valueCounter.processTuple(new InternalCountOutput<Integer>(key, count, valSet));
- }
-
- @Test
- public void testSetup() throws Exception
- {
- insertValues();
- Statement stmt = valueCounter.getStore().getConnection().createStatement();
- com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap attributes = new com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap();
- attributes.put(DAG.APPLICATION_ID, APP_ID);
- attributes.put(DAG.APPLICATION_PATH, applicationPath);
- attributes.put(OperatorContext.ACTIVATION_WINDOW_ID, 2L);
-
- valueCounter.setup(new OperatorContextTestHelper.TestIdOperatorContext(0, attributes));
-
- ResultSet resultSet = stmt.executeQuery("SELECT col2 FROM " + TABLE_NAME + " WHERE col1 = 2");
- ArrayList<Integer> answersAfterClear = new ArrayList<Integer>();
- for (int i = 3; i < 16; i += 3) {
- answersAfterClear.add(i);
- }
- Assert.assertEquals(answersAfterClear, processResult(resultSet));
-
- resultSet = stmt.executeQuery("SELECT col2 FROM " + TABLE_NAME + " WHERE col1 = 3");
- ArrayList<Integer> answersThree = new ArrayList<Integer>();
- answersThree.add(2);
- answersThree.add(4);
- answersThree.add(6);
- answersThree.add(8);
- answersThree.add(10);
- for (int i = 11; i < 21; i++) {
- answersThree.add(i);
- }
- Assert.assertEquals(answersThree, processResult(resultSet));
- stmt.executeQuery("DELETE FROM " + TABLE_NAME);
- }
-
- @BeforeClass
- public static void setup() throws Exception
- {
- valueCounter = new IntegerUniqueValueCountAppender();
- Class.forName(INMEM_DB_DRIVER).newInstance();
- Connection con = DriverManager.getConnection(INMEM_DB_URL, new Properties());
- Statement stmt = con.createStatement();
- stmt.execute("CREATE TABLE IF NOT EXISTS " + TABLE_NAME + " (col1 INTEGER, col2 INTEGER, col3 BIGINT)");
- com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap attributes = new com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap();
- attributes.put(DAG.APPLICATION_ID, APP_ID);
- attributes.put(DAG.APPLICATION_PATH, applicationPath);
- attributes.put(OperatorContext.ACTIVATION_WINDOW_ID, 0L);
- valueCounter.setTableName(TABLE_NAME);
- valueCounter.getStore().setDatabaseDriver(INMEM_DB_DRIVER);
- valueCounter.getStore().setDatabaseUrl(INMEM_DB_URL);
- TestIdOperatorContext context = new OperatorContextTestHelper.TestIdOperatorContext(OPERATOR_ID, attributes);
- valueCounter.setup(context);
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/distributedistinct/src/test/java/com/datatorrent/demos/distributeddistinct/StatefulApplicationTest.java
----------------------------------------------------------------------
diff --git a/demos/distributedistinct/src/test/java/com/datatorrent/demos/distributeddistinct/StatefulApplicationTest.java b/demos/distributedistinct/src/test/java/com/datatorrent/demos/distributeddistinct/StatefulApplicationTest.java
deleted file mode 100644
index 57ac964..0000000
--- a/demos/distributedistinct/src/test/java/com/datatorrent/demos/distributeddistinct/StatefulApplicationTest.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.distributeddistinct;
-
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.Properties;
-
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.LocalMode;
-
-public class StatefulApplicationTest
-{
-
- @BeforeClass
- public static void setup()
- {
- try {
- Class.forName(StatefulUniqueCountTest.INMEM_DB_DRIVER).newInstance();
- Connection con = DriverManager.getConnection(StatefulUniqueCountTest.INMEM_DB_URL, new Properties());
- Statement stmt = con.createStatement();
- stmt.execute("CREATE TABLE IF NOT EXISTS " + StatefulUniqueCountTest.TABLE_NAME + " (col1 INTEGER, col2 INTEGER, col3 BIGINT)");
- } catch (InstantiationException e) {
- throw new RuntimeException(e);
- } catch (IllegalAccessException e) {
- throw new RuntimeException(e);
- } catch (ClassNotFoundException e) {
- throw new RuntimeException(e);
- } catch (SQLException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Test
- public void testApplication() throws Exception
- {
- LocalMode lma = LocalMode.newInstance();
- Configuration conf = new Configuration(false);
- conf.set("dt.operator.StatefulUniqueCounter.prop.tableName", "Test_Lookup_Cache");
- conf.set("dt.operator.StatefulUniqueCounter.prop.store.dbUrl", "jdbc:hsqldb:mem:test;sql.syntax_mys=true");
- conf.set("dt.operator.StatefulUniqueCounter.prop.store.dbDriver", "org.hsqldb.jdbcDriver");
-
- lma.prepareDAG(new StatefulApplication(), conf);
- lma.cloneDAG();
- LocalMode.Controller lc = lma.getController();
- lc.setHeartbeatMonitoringEnabled(false);
- lc.runAsync();
-
- long now = System.currentTimeMillis();
- while (System.currentTimeMillis() - now < 15000) {
- Thread.sleep(1000);
- }
-
- lc.shutdown();
- }
-}
[02/30] apex-malhar git commit: Renamed demos to examples. Packages
and artifactid names are changed as suggested.
Posted by th...@apache.org.
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/twitter/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/examples/twitter/src/main/resources/META-INF/properties.xml b/examples/twitter/src/main/resources/META-INF/properties.xml
new file mode 100644
index 0000000..3f6e74d
--- /dev/null
+++ b/examples/twitter/src/main/resources/META-INF/properties.xml
@@ -0,0 +1,121 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+
+<!-- properties for rolling top words example -->
+<configuration>
+ <property>
+ <name>dt.attr.MASTER_MEMORY_MB</name>
+ <value>1256</value>
+ </property>
+ <!-- default operator size 256MB -->
+ <property>
+ <name>dt.application.*.operator.*.attr.MEMORY_MB</name>
+ <value>256</value>
+ </property>
+ <property>
+ <name>dt.application.*.operator.*.attr.JVM_OPTIONS</name>
+ <value>-Xmx128M</value>
+ </property>
+
+ <!-- default buffer memory 256MB -->
+ <property>
+ <name>dt.application.*.operator.*.port.*.attr.BUFFER_MEMORY_MB</name>
+ <value>256</value>
+ </property>
+ <property>
+ <name>dt.operator.TweetSampler.consumerKey</name>
+ </property>
+ <property>
+ <name>dt.operator.TweetSampler.consumerSecret</name>
+ </property>
+ <property>
+ <name>dt.operator.TweetSampler.accessToken</name>
+ </property>
+ <property>
+ <name>dt.operator.TweetSampler.accessTokenSecret</name>
+ </property>
+ <property>
+ <name>dt.operator.TweetSampler.feedMultiplierVariance</name>
+ <value>5</value>
+ </property>
+ <property>
+ <name>dt.operator.TweetSampler.feedMultiplier</name>
+ <value>20</value>
+ </property>
+
+ <!-- RollingTopWordsExample -->
+
+ <property>
+ <name>dt.application.RollingTopWordsExample.operator.TopCounter.topCount</name>
+ <value>10</value>
+ </property>
+ <property>
+ <name>dt.application.RollingTopWordsExample.stream.TweetStream.locality</name>
+ <value>CONTAINER_LOCAL</value>
+ </property>
+ <property>
+ <name>dt.application.RollingTopWordsExample.operator.SnapshotServer.embeddableQueryInfoProvider.topic</name>
+ <value>TwitterWordsQuery</value>
+ </property>
+ <property>
+ <name>dt.application.RollingTopWordsExample.operator.QueryResult.topic</name>
+ <value>TwitterWordsQueryResult</value>
+ </property>
+ <property>
+ <name>dt.application.RollingTopWordsExample.operator.QueryResult.numRetries</name>
+ <value>2147483647</value>
+ </property>
+
+ <!-- TwitterExample -->
+
+ <property>
+ <name>dt.application.TwitterExample.operator.SnapshotServer.embeddableQueryInfoProvider.topic</name>
+ <value>TwitterURLQuery</value>
+ </property>
+ <property>
+ <name>dt.application.TwitterExample.operator.QueryResult.topic</name>
+ <value>TwitterURLQueryResult</value>
+ </property>
+ <property>
+ <name>dt.application.TwitterExample.operator.QueryResult.numRetries</name>
+ <value>2147483647</value>
+ </property>
+ <property>
+ <name>dt.application.TwitterExample.operator.UniqueURLCounter.attr.APPLICATION_WINDOW_COUNT</name>
+ <value>60</value>
+ </property>
+
+ <!-- TwitterTrendingExample -->
+
+ <property>
+ <name>dt.application.TwitterTrendingExample.operator.SnapshotServer.embeddableQueryInfoProvider.topic</name>
+ <value>TwitterHashtagQueryExample</value>
+ </property>
+ <property>
+ <name>dt.application.TwitterTrendingExample.operator.QueryResult.topic</name>
+ <value>TwitterHashtagQueryResultExample</value>
+ </property>
+ <property>
+ <name>dt.application.TwitterTrendingExample.operator.QueryResult.numRetries</name>
+ <value>2147483647</value>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/twitter/src/main/resources/mysql.sql
----------------------------------------------------------------------
diff --git a/examples/twitter/src/main/resources/mysql.sql b/examples/twitter/src/main/resources/mysql.sql
new file mode 100644
index 0000000..e0b97dd
--- /dev/null
+++ b/examples/twitter/src/main/resources/mysql.sql
@@ -0,0 +1,35 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+--
+
+DROP TABLE if exists tweets;
+CREATE TABLE tweets (
+ window_id LONG NOT NULL,
+ creation_date DATE,
+ text VARCHAR(256) NOT NULL,
+ userid VARCHAR(40) NOT NULL,
+ KEY ( userid, creation_date)
+ );
+
+drop table if exists dt_window_id_tracker;
+CREATE TABLE dt_window_id_tracker (
+ dt_application_id VARCHAR(100) NOT NULL,
+ dt_operator_id int(11) NOT NULL,
+ dt_window_id bigint NOT NULL,
+ UNIQUE (dt_application_id, dt_operator_id, dt_window_id)
+) ENGINE=MyISAM DEFAULT CHARSET=latin1;
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/twitter/src/main/resources/top_urls.tplg.properties
----------------------------------------------------------------------
diff --git a/examples/twitter/src/main/resources/top_urls.tplg.properties b/examples/twitter/src/main/resources/top_urls.tplg.properties
new file mode 100644
index 0000000..c106d7d
--- /dev/null
+++ b/examples/twitter/src/main/resources/top_urls.tplg.properties
@@ -0,0 +1,48 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+stram.node.twitterfeed.classname=com.datatorrent.example.twitter.TwitterSampleInput
+
+stram.stream.status.source=twitterfeed.output
+stram.stream.status.sinks=urlextractor.input
+
+stram.node.urlextractor.classname=com.datatorrent.example.twitter.TwitterStatusURLExtractor
+
+stram.stream.collapsedurls.source=urlextractor.output
+stram.stream.collapsedurls.sinks=
+
+stram.node.
+stram.stream.partitionedtf.input=twitterfeed.output
+stram.stream.partitionedtf.output=partitioned_counter.input
+stram.stream.partitionedtf.serdeClassname=com.datatorrent.example.twitter.URLSerDe
+
+stram.node.partitioned_counter.classname=com.datatorrent.example.twitter.PartitionedCounter
+stram.node.partitioned_counter.topCount=10
+
+stram.stream.merge_stream.input=partitioned_counter.output
+stram.stream.merge_stream.output=merge_counter.input
+stram.stream.merge_stream.serdeClassname=com.datatorrent.example.twitter.URLHolderSerde
+
+stram.node.merge_counter.classname=com.datatorrent.example.twitter.MergeSorter
+stram.node.merge_counter.topCount=10
+
+stram.stream.merged_stream.input=merge_counter.output
+stram.stream.merged_stream.output=console.input
+
+stram.node.console.classname=com.datatorrent.stream.ConsoleOutputStream
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/twitter/src/main/resources/twitterHashTagDataSchema.json
----------------------------------------------------------------------
diff --git a/examples/twitter/src/main/resources/twitterHashTagDataSchema.json b/examples/twitter/src/main/resources/twitterHashTagDataSchema.json
new file mode 100644
index 0000000..0c9296c
--- /dev/null
+++ b/examples/twitter/src/main/resources/twitterHashTagDataSchema.json
@@ -0,0 +1,4 @@
+{
+ "values": [{"name": "hashtag", "type": "string"},
+ {"name": "count", "type": "integer"}]
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/twitter/src/main/resources/twitterURLDataSchema.json
----------------------------------------------------------------------
diff --git a/examples/twitter/src/main/resources/twitterURLDataSchema.json b/examples/twitter/src/main/resources/twitterURLDataSchema.json
new file mode 100644
index 0000000..ecf723e
--- /dev/null
+++ b/examples/twitter/src/main/resources/twitterURLDataSchema.json
@@ -0,0 +1,4 @@
+{
+ "values": [{"name": "url", "type": "string"},
+ {"name": "count", "type": "integer"}]
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/twitter/src/main/resources/twitterWordDataSchema.json
----------------------------------------------------------------------
diff --git a/examples/twitter/src/main/resources/twitterWordDataSchema.json b/examples/twitter/src/main/resources/twitterWordDataSchema.json
new file mode 100644
index 0000000..5e8e7c0
--- /dev/null
+++ b/examples/twitter/src/main/resources/twitterWordDataSchema.json
@@ -0,0 +1,4 @@
+{
+ "values": [{"name": "word", "type": "string"},
+ {"name": "count", "type": "integer"}]
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/twitter/src/test/java/org/apache/apex/examples/twitter/TwitterDumpApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/twitter/src/test/java/org/apache/apex/examples/twitter/TwitterDumpApplicationTest.java b/examples/twitter/src/test/java/org/apache/apex/examples/twitter/TwitterDumpApplicationTest.java
new file mode 100644
index 0000000..fcde9ef
--- /dev/null
+++ b/examples/twitter/src/test/java/org/apache/apex/examples/twitter/TwitterDumpApplicationTest.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.twitter;
+
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+
+import static org.junit.Assert.assertEquals;
+
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.LocalMode;
+
+/**
+ * Test for the application which taps into the twitter's sample input stream and
+ * dumps all the tweets into a database.
+ */
+public class TwitterDumpApplicationTest
+{
+ @Test
+ public void testPopulateDAG() throws Exception
+ {
+ Configuration configuration = new Configuration(false);
+
+ LocalMode lm = LocalMode.newInstance();
+ DAG prepareDAG = lm.prepareDAG(new TwitterDumpApplication(), configuration);
+ DAG clonedDAG = lm.cloneDAG();
+
+ assertEquals("Serialization", prepareDAG, clonedDAG);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/twitter/src/test/java/org/apache/apex/examples/twitter/TwitterTopCounterTest.java
----------------------------------------------------------------------
diff --git a/examples/twitter/src/test/java/org/apache/apex/examples/twitter/TwitterTopCounterTest.java b/examples/twitter/src/test/java/org/apache/apex/examples/twitter/TwitterTopCounterTest.java
new file mode 100644
index 0000000..628a807
--- /dev/null
+++ b/examples/twitter/src/test/java/org/apache/apex/examples/twitter/TwitterTopCounterTest.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.twitter;
+
+import org.junit.Test;
+import org.apache.hadoop.conf.Configuration;
+import com.datatorrent.api.LocalMode;
+import com.datatorrent.contrib.twitter.TwitterSampleInput;
+
+/**
+ * Test the DAG declaration in local mode.
+ */
+public class TwitterTopCounterTest
+{
+ /**
+ * This test requires twitter authentication setup and is skipped by default
+ * (see {@link TwitterSampleInput}).
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testApplication() throws Exception
+ {
+ LocalMode lma = LocalMode.newInstance();
+ new TwitterTopCounterApplication().populateDAG(lma.getDAG(), new Configuration(false));
+ LocalMode.Controller lc = lma.getController();
+ lc.run(120000);
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/twitter/src/test/java/org/apache/apex/examples/twitter/TwitterTopWordsTest.java
----------------------------------------------------------------------
diff --git a/examples/twitter/src/test/java/org/apache/apex/examples/twitter/TwitterTopWordsTest.java b/examples/twitter/src/test/java/org/apache/apex/examples/twitter/TwitterTopWordsTest.java
new file mode 100644
index 0000000..e888eb3
--- /dev/null
+++ b/examples/twitter/src/test/java/org/apache/apex/examples/twitter/TwitterTopWordsTest.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.twitter;
+
+import org.junit.Test;
+import org.apache.hadoop.conf.Configuration;
+import com.datatorrent.api.LocalMode;
+import com.datatorrent.contrib.twitter.TwitterSampleInput;
+
+/**
+ * Test the DAG declaration in local mode.
+ */
+public class TwitterTopWordsTest
+{
+ /**
+ * This test requires twitter authentication setup and is skipped by default
+ * (see {@link TwitterSampleInput}).
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testApplication() throws Exception
+ {
+ TwitterTopWordsApplication app = new TwitterTopWordsApplication();
+ Configuration conf = new Configuration(false);
+ conf.addResource("dt-site-rollingtopwords.xml");
+ LocalMode lma = LocalMode.newInstance();
+ lma.prepareDAG(app, conf);
+ LocalMode.Controller lc = lma.getController();
+ lc.run(120000);
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/twitter/src/test/resources/dt-site-rollingtopwords.xml
----------------------------------------------------------------------
diff --git a/examples/twitter/src/test/resources/dt-site-rollingtopwords.xml b/examples/twitter/src/test/resources/dt-site-rollingtopwords.xml
new file mode 100644
index 0000000..fa9b534
--- /dev/null
+++ b/examples/twitter/src/test/resources/dt-site-rollingtopwords.xml
@@ -0,0 +1,73 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<configuration>
+<property>
+ <name>dt.attr.MASTER_MEMORY_MB</name>
+ <value>1024</value>
+ </property>
+ <property>
+ <name>dt.application.RollingTopWordsExample.class</name>
+ <value>org.apache.apex.examples.twitter.TwitterTopWordsApplication</value>
+ <description>An alias for the application</description>
+ </property>
+ <property>
+ <name>dt.application.RollingTopWordsExample.operator.TopCounter.topCount
+ </name>
+ <value>10</value>
+ </property>
+ <property>
+ <name>dt.application.RollingTopWordsExample.stream.TweetStream.locality
+ </name>
+ <value>CONTAINER_LOCAL</value>
+ </property>
+ <property>
+ <name>dt.application.RollingTopWordsExample.stream.TwittedWords.locality
+ </name>
+ <value></value>
+ </property>
+ <property>
+ <name>dt.application.RollingTopWordsExample.stream.UniqueWordCounts.locality
+ </name>
+ <value></value>
+ </property>
+ <property>
+ <name>dt.application.RollingTopWordsExample.stream.TopWords.locality</name>
+ <value></value>
+ </property>
+ <property>
+ <name>dt.operator.TweetSampler.consumerKey</name>
+ <value>r1DqM35iCTjbgLHf1R7rDbF5R</value>
+ </property>
+ <property>
+ <name>dt.operator.TweetSampler.consumerSecret</name>
+ <value>KBpZiR0glPzvZPm1Sa7sq9MCGQ2H2DrVChNtmYQcU75fwuHjed</value>
+ </property>
+ <property>
+ <name>dt.operator.TweetSampler.accessToken</name>
+ <value>2490355837-lXKev9vIGzftDjDu9LlyQiuGAqjYyPELFRyRpQo</value>
+ </property>
+ <property>
+ <name>dt.operator.TweetSampler.accessTokenSecret</name>
+ <value>lmAxZlFhcBqeTxmyatFT43fzshzrv6lsOAtHDsCBLjiuk</value>
+ </property>
+
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/twitter/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/examples/twitter/src/test/resources/log4j.properties b/examples/twitter/src/test/resources/log4j.properties
new file mode 100644
index 0000000..cf0d19e
--- /dev/null
+++ b/examples/twitter/src/test/resources/log4j.properties
@@ -0,0 +1,43 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+log4j.rootLogger=DEBUG,CONSOLE
+
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+log4j.appender.CONSOLE.threshold=${test.log.console.threshold}
+test.log.console.threshold=DEBUG
+
+log4j.appender.RFA=org.apache.log4j.RollingFileAppender
+log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
+log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+log4j.appender.RFA.File=/tmp/app.log
+
+# to enable, add SYSLOG to rootLogger
+log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender
+log4j.appender.SYSLOG.syslogHost=127.0.0.1
+log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout
+log4j.appender.SYSLOG.layout.conversionPattern=${dt.cid} %-5p [%t] %c{2} %x - %m%n
+log4j.appender.SYSLOG.Facility=LOCAL1
+
+log4j.logger.org=info
+#log4j.logger.org.apache.commons.beanutils=warn
+log4j.logger.com.datatorrent=debug
+log4j.logger.org.apache.apex=debug
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/uniquecount/pom.xml
----------------------------------------------------------------------
diff --git a/examples/uniquecount/pom.xml b/examples/uniquecount/pom.xml
new file mode 100644
index 0000000..37e4bb1
--- /dev/null
+++ b/examples/uniquecount/pom.xml
@@ -0,0 +1,50 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>malhar-examples-uniquecount</artifactId>
+ <packaging>jar</packaging>
+
+ <name>Apache Apex Malhar Unique Count Example</name>
+ <description></description>
+
+ <parent>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-examples</artifactId>
+ <version>3.7.0-SNAPSHOT</version>
+ </parent>
+
+ <properties>
+ <skipTests>true</skipTests>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ <version>3.1</version>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/uniquecount/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/examples/uniquecount/src/assemble/appPackage.xml b/examples/uniquecount/src/assemble/appPackage.xml
new file mode 100644
index 0000000..4138cf2
--- /dev/null
+++ b/examples/uniquecount/src/assemble/appPackage.xml
@@ -0,0 +1,59 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+ <id>appPackage</id>
+ <formats>
+ <format>jar</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>${basedir}/target/</directory>
+ <outputDirectory>/app</outputDirectory>
+ <includes>
+ <include>${project.artifactId}-${project.version}.jar</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/target/deps</directory>
+ <outputDirectory>/lib</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/site/conf</directory>
+ <outputDirectory>/conf</outputDirectory>
+ <includes>
+ <include>*.xml</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/META-INF</directory>
+ <outputDirectory>/META-INF</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/app</directory>
+ <outputDirectory>/app</outputDirectory>
+ </fileSet>
+ </fileSets>
+
+</assembly>
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/uniquecount/src/main/java/org/apache/apex/examples/uniquecount/Application.java
----------------------------------------------------------------------
diff --git a/examples/uniquecount/src/main/java/org/apache/apex/examples/uniquecount/Application.java b/examples/uniquecount/src/main/java/org/apache/apex/examples/uniquecount/Application.java
new file mode 100644
index 0000000..fad3718
--- /dev/null
+++ b/examples/uniquecount/src/main/java/org/apache/apex/examples/uniquecount/Application.java
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.uniquecount;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.Context;
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.DAG.Locality;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+
+import com.datatorrent.common.partitioner.StatelessPartitioner;
+import com.datatorrent.lib.algo.UniqueCounter;
+import com.datatorrent.lib.converter.MapToKeyHashValuePairConverter;
+import com.datatorrent.lib.io.ConsoleOutputOperator;
+import com.datatorrent.lib.stream.Counter;
+import com.datatorrent.lib.stream.StreamDuplicater;
+import com.datatorrent.lib.util.KeyHashValPair;
+
+/**
+ * Application to demonstrate PartitionableUniqueCount operator. <br>
+ * The input operator generate random keys, which is sent to
+ * PartitionableUniqueCount operator initially partitioned into three partitions to
+ * test unifier functionality, and output of the operator is sent to verifier to verify
+ * that it generates correct result.
+ *
+ * @since 1.0.2
+ */
+@ApplicationAnnotation(name = "UniqueValueCountExample")
+public class Application implements StreamingApplication
+{
+
+ @Override
+ public void populateDAG(DAG dag, Configuration entries)
+ {
+ /* Generate random key-value pairs */
+ RandomKeysGenerator randGen = dag.addOperator("randomgen", new RandomKeysGenerator());
+
+
+ /* Initialize with three partition to start with */
+ // UniqueCount1 uniqCount = dag.addOperator("uniqevalue", new UniqueCount1());
+ UniqueCounter<Integer> uniqCount = dag.addOperator("uniqevalue", new UniqueCounter<Integer>());
+
+ MapToKeyHashValuePairConverter<Integer, Integer> converter = dag.addOperator("converter", new MapToKeyHashValuePairConverter());
+
+ uniqCount.setCumulative(false);
+ dag.setAttribute(uniqCount, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<UniqueCounter<Integer>>(3));
+
+ CountVerifier<Integer> verifier = dag.addOperator("verifier", new CountVerifier<Integer>());
+ StreamDuplicater<KeyHashValPair<Integer, Integer>> dup = dag.addOperator("dup", new StreamDuplicater<KeyHashValPair<Integer, Integer>>());
+ ConsoleOutputOperator output = dag.addOperator("output", new ConsoleOutputOperator());
+
+ ConsoleOutputOperator successOutput = dag.addOperator("successoutput", new ConsoleOutputOperator());
+ successOutput.setStringFormat("Success %d");
+ ConsoleOutputOperator failureOutput = dag.addOperator("failureoutput", new ConsoleOutputOperator());
+ failureOutput.setStringFormat("Failure %d");
+
+ // success and failure counters.
+ Counter successcounter = dag.addOperator("successcounter", new Counter());
+ Counter failurecounter = dag.addOperator("failurecounter", new Counter());
+
+ dag.addStream("datain", randGen.outPort, uniqCount.data);
+ dag.addStream("dataverification0", randGen.verificationPort, verifier.in1);
+ dag.addStream("convert", uniqCount.count, converter.input).setLocality(Locality.THREAD_LOCAL);
+ dag.addStream("split", converter.output, dup.data);
+ dag.addStream("consoutput", dup.out1, output.input);
+ dag.addStream("dataverification1", dup.out2, verifier.in2);
+ dag.addStream("successc", verifier.successPort, successcounter.input);
+ dag.addStream("failurec", verifier.failurePort, failurecounter.input);
+ dag.addStream("succconsoutput", successcounter.output, successOutput.input);
+ dag.addStream("failconsoutput", failurecounter.output, failureOutput.input);
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/uniquecount/src/main/java/org/apache/apex/examples/uniquecount/CountVerifier.java
----------------------------------------------------------------------
diff --git a/examples/uniquecount/src/main/java/org/apache/apex/examples/uniquecount/CountVerifier.java b/examples/uniquecount/src/main/java/org/apache/apex/examples/uniquecount/CountVerifier.java
new file mode 100644
index 0000000..422807e
--- /dev/null
+++ b/examples/uniquecount/src/main/java/org/apache/apex/examples/uniquecount/CountVerifier.java
@@ -0,0 +1,107 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.uniquecount;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import com.datatorrent.api.Context;
+import com.datatorrent.api.DefaultInputPort;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.Operator;
+import com.datatorrent.api.annotation.OutputPortFieldAnnotation;
+import com.datatorrent.lib.util.KeyHashValPair;
+
+/*
+Compare results and print non-matching values to console.
+ */
+/**
+ * <p>CountVerifier class.</p>
+ *
+ * @since 1.0.2
+ */
+public class CountVerifier<K> implements Operator
+{
+ HashMap<K, Integer> map1 = new HashMap<K, Integer>();
+ HashMap<K, Integer> map2 = new HashMap<K, Integer>();
+
+ public final transient DefaultInputPort<KeyHashValPair<K, Integer>> in1 = new DefaultInputPort<KeyHashValPair<K, Integer>>()
+ {
+ @Override
+ public void process(KeyHashValPair<K, Integer> tuple)
+ {
+ processTuple(tuple, map1);
+ }
+ };
+
+ public final transient DefaultInputPort<KeyHashValPair<K, Integer>> in2 = new DefaultInputPort<KeyHashValPair<K, Integer>>()
+ {
+ @Override
+ public void process(KeyHashValPair<K, Integer> tuple)
+ {
+ processTuple(tuple, map2);
+ }
+ };
+
+ void processTuple(KeyHashValPair<K, Integer> tuple, HashMap<K, Integer> map)
+ {
+ map.put(tuple.getKey(), tuple.getValue());
+ }
+
+ @OutputPortFieldAnnotation(optional = true)
+ public final transient DefaultOutputPort<Integer> successPort = new DefaultOutputPort<Integer>();
+ @OutputPortFieldAnnotation(optional = true)
+ public final transient DefaultOutputPort<Integer> failurePort = new DefaultOutputPort<Integer>();
+
+ @Override
+ public void beginWindow(long l)
+ {
+
+ }
+
+ @Override
+ public void endWindow()
+ {
+ int failureCount = 0;
+ for (Map.Entry<K, Integer> e : map1.entrySet()) {
+ K key = e.getKey();
+ int val = map2.get(key);
+ if (val != e.getValue()) {
+ failureCount++;
+ }
+ }
+ if (failureCount != 0) {
+ failurePort.emit(failureCount);
+ } else {
+ successPort.emit(map1.size());
+ }
+ }
+
+ @Override
+ public void setup(Context.OperatorContext operatorContext)
+ {
+
+ }
+
+ @Override
+ public void teardown()
+ {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/uniquecount/src/main/java/org/apache/apex/examples/uniquecount/RandomDataGenerator.java
----------------------------------------------------------------------
diff --git a/examples/uniquecount/src/main/java/org/apache/apex/examples/uniquecount/RandomDataGenerator.java b/examples/uniquecount/src/main/java/org/apache/apex/examples/uniquecount/RandomDataGenerator.java
new file mode 100644
index 0000000..3e29a70
--- /dev/null
+++ b/examples/uniquecount/src/main/java/org/apache/apex/examples/uniquecount/RandomDataGenerator.java
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.uniquecount;
+
+import java.util.HashMap;
+import java.util.Random;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.datatorrent.api.Context;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.InputOperator;
+import com.datatorrent.lib.util.KeyValPair;
+
+/**
+ * Generate random Key value pairs.
+ * key is string and value is int, it emits the pair as KeyValPair on outPort,
+ *
+ * @since 1.0.2
+ */
+public class RandomDataGenerator implements InputOperator
+{
+ public final transient DefaultOutputPort<KeyValPair<String, Object>> outPort = new DefaultOutputPort<KeyValPair<String, Object>>();
+ private HashMap<String, Integer> dataInfo;
+ private final transient Logger LOG = LoggerFactory.getLogger(RandomDataGenerator.class);
+ private int count;
+ private int sleepMs = 10;
+ private int keyRange = 100;
+ private int valueRange = 100;
+ private long tupleBlast = 10000;
+ private Random random;
+
+ public RandomDataGenerator()
+ {
+ random = new Random();
+ }
+
+ @Override
+ public void emitTuples()
+ {
+ for (int i = 0; i < tupleBlast; i++) {
+ String key = String.valueOf(random.nextInt(keyRange));
+ int val = random.nextInt(valueRange);
+ outPort.emit(new KeyValPair<String, Object>(key, val));
+ }
+ try {
+ Thread.sleep(sleepMs);
+ } catch (Exception ex) {
+ LOG.error(ex.getMessage());
+ }
+ count++;
+ }
+
+ public int getSleepMs()
+ {
+ return sleepMs;
+ }
+
+ public void setSleepMs(int sleepMs)
+ {
+ this.sleepMs = sleepMs;
+ }
+
+ public long getTupleBlast()
+ {
+ return tupleBlast;
+ }
+
+ public void setTupleBlast(long tupleBlast)
+ {
+ this.tupleBlast = tupleBlast;
+ }
+
+ @Override
+ public void beginWindow(long l)
+ {
+
+ }
+
+ @Override
+ public void endWindow()
+ {
+ LOG.debug("emitTuples called " + count + " times in this window");
+ count = 0;
+ }
+
+ @Override
+ public void setup(Context.OperatorContext operatorContext)
+ {
+
+ }
+
+ @Override
+ public void teardown()
+ {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/uniquecount/src/main/java/org/apache/apex/examples/uniquecount/RandomKeyValues.java
----------------------------------------------------------------------
diff --git a/examples/uniquecount/src/main/java/org/apache/apex/examples/uniquecount/RandomKeyValues.java b/examples/uniquecount/src/main/java/org/apache/apex/examples/uniquecount/RandomKeyValues.java
new file mode 100644
index 0000000..6d39c37
--- /dev/null
+++ b/examples/uniquecount/src/main/java/org/apache/apex/examples/uniquecount/RandomKeyValues.java
@@ -0,0 +1,153 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.uniquecount;
+
+import java.util.BitSet;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Random;
+
+import com.datatorrent.api.Context.OperatorContext;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.InputOperator;
+import com.datatorrent.lib.util.KeyValPair;
+
+/**
+ * Input port operator for generating random values on keys. <br>
+ * Key(s) : key + integer in range between 0 and numKeys <br>
+ * Value(s) : integer in range of 0 to numValuesPerKeys <br>
+ *
+ * @since 0.9.3
+ */
+public class RandomKeyValues implements InputOperator
+{
+ public final transient DefaultOutputPort<KeyValPair<String, Object>> outport = new DefaultOutputPort<KeyValPair<String, Object>>();
+ private Random random = new Random(11111);
+ private int numKeys;
+ private int numValuesPerKeys;
+ private int tuppleBlast = 1000;
+ private int emitDelay = 20; /* 20 ms */
+
+ /* For verification */
+ private Map<Integer, BitSet> history = new HashMap<Integer, BitSet>();
+
+ public RandomKeyValues()
+ {
+ this.numKeys = 100;
+ this.numValuesPerKeys = 100;
+ }
+
+ public RandomKeyValues(int keys, int values)
+ {
+ this.numKeys = keys;
+ this.numValuesPerKeys = values;
+ }
+
+ @Override
+ public void beginWindow(long windowId)
+ {
+ }
+
+ @Override
+ public void endWindow()
+ {
+ }
+
+ @Override
+ public void setup(OperatorContext context)
+ {
+ }
+
+ @Override
+ public void teardown()
+ {
+ }
+
+ @Override
+ public void emitTuples()
+ {
+ /* generate tuples randomly, */
+ for (int i = 0; i < tuppleBlast; i++) {
+ int intKey = random.nextInt(numKeys);
+ String key = "key" + String.valueOf(intKey);
+ int value = random.nextInt(numValuesPerKeys);
+
+ // update history for verifying later.
+ BitSet bmap = history.get(intKey);
+ if (bmap == null) {
+ bmap = new BitSet();
+ history.put(intKey, bmap);
+ }
+ bmap.set(value);
+
+ // emit the key with value.
+ outport.emit(new KeyValPair<String, Object>(key, value));
+ }
+ try {
+ Thread.sleep(emitDelay);
+ } catch (Exception e) {
+ // Ignore.
+ }
+ }
+
+ public int getNumKeys()
+ {
+ return numKeys;
+ }
+
+ public void setNumKeys(int numKeys)
+ {
+ this.numKeys = numKeys;
+ }
+
+ public int getNumValuesPerKeys()
+ {
+ return numValuesPerKeys;
+ }
+
+ public void setNumValuesPerKeys(int numValuesPerKeys)
+ {
+ this.numValuesPerKeys = numValuesPerKeys;
+ }
+
+ public int getTuppleBlast()
+ {
+ return tuppleBlast;
+ }
+
+ public void setTuppleBlast(int tuppleBlast)
+ {
+ this.tuppleBlast = tuppleBlast;
+ }
+
+ public int getEmitDelay()
+ {
+ return emitDelay;
+ }
+
+ public void setEmitDelay(int emitDelay)
+ {
+ this.emitDelay = emitDelay;
+ }
+
+ public void debug()
+ {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/uniquecount/src/main/java/org/apache/apex/examples/uniquecount/RandomKeysGenerator.java
----------------------------------------------------------------------
diff --git a/examples/uniquecount/src/main/java/org/apache/apex/examples/uniquecount/RandomKeysGenerator.java b/examples/uniquecount/src/main/java/org/apache/apex/examples/uniquecount/RandomKeysGenerator.java
new file mode 100644
index 0000000..9f26b4e
--- /dev/null
+++ b/examples/uniquecount/src/main/java/org/apache/apex/examples/uniquecount/RandomKeysGenerator.java
@@ -0,0 +1,153 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.uniquecount;
+
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.commons.lang3.mutable.MutableInt;
+
+import com.datatorrent.api.Context;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.InputOperator;
+import com.datatorrent.api.annotation.OutputPortFieldAnnotation;
+import com.datatorrent.lib.util.KeyHashValPair;
+
+/*
+ Generate random keys.
+ */
+/**
+ * <p>RandomKeysGenerator class.</p>
+ *
+ * @since 1.0.2
+ */
+public class RandomKeysGenerator implements InputOperator
+{
+
+ protected int numKeys = 100;
+ protected int tupleBlast = 15000;
+ protected long sleepTime = 0;
+ protected Map<Integer, MutableInt> history = new HashMap<Integer, MutableInt>();
+ private Random random = new Random();
+ private Date date = new Date();
+ private long start;
+
+ @OutputPortFieldAnnotation(optional = false)
+ public transient DefaultOutputPort<Integer> outPort = new DefaultOutputPort<Integer>();
+
+ @OutputPortFieldAnnotation(optional = true)
+ public transient DefaultOutputPort<KeyHashValPair<Integer, Integer>> verificationPort =
+ new DefaultOutputPort<KeyHashValPair<Integer, Integer>>();
+
+ @Override
+ public void emitTuples()
+ {
+ for (int i = 0; i < tupleBlast; i++) {
+ int key = random.nextInt(numKeys);
+ outPort.emit(key);
+
+
+ if (verificationPort.isConnected()) {
+ // maintain history for later verification.
+ MutableInt count = history.get(key);
+ if (count == null) {
+ count = new MutableInt(0);
+ history.put(key, count);
+ }
+ count.increment();
+ }
+
+ }
+ try {
+ if (sleepTime != 0) {
+ Thread.sleep(sleepTime);
+ }
+ } catch (Exception ex) {
+ // Ignore.
+ }
+ }
+
+ public RandomKeysGenerator()
+ {
+ start = date.getTime();
+ }
+
+ @Override
+ public void beginWindow(long l)
+ {
+
+ }
+
+ @Override
+ public void endWindow()
+ {
+
+ if (verificationPort.isConnected()) {
+ for (Map.Entry<Integer, MutableInt> e : history.entrySet()) {
+ verificationPort.emit(new KeyHashValPair<Integer, Integer>(e.getKey(), e.getValue().toInteger()));
+ }
+ history.clear();
+ }
+
+ }
+
+ @Override
+ public void setup(Context.OperatorContext operatorContext)
+ {
+
+ }
+
+ @Override
+ public void teardown()
+ {
+
+ }
+
+ public int getNumKeys()
+ {
+ return numKeys;
+ }
+
+ public void setNumKeys(int numKeys)
+ {
+ this.numKeys = numKeys;
+ }
+
+ public int getTupleBlast()
+ {
+ return tupleBlast;
+ }
+
+ public void setTupleBlast(int tupleBlast)
+ {
+ this.tupleBlast = tupleBlast;
+ }
+
+ public long getSleepTime()
+ {
+ return sleepTime;
+ }
+
+ public void setSleepTime(long sleepTime)
+ {
+ this.sleepTime = sleepTime;
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/uniquecount/src/main/java/org/apache/apex/examples/uniquecount/UniqueKeyValCountExample.java
----------------------------------------------------------------------
diff --git a/examples/uniquecount/src/main/java/org/apache/apex/examples/uniquecount/UniqueKeyValCountExample.java b/examples/uniquecount/src/main/java/org/apache/apex/examples/uniquecount/UniqueKeyValCountExample.java
new file mode 100644
index 0000000..ff9fdf0
--- /dev/null
+++ b/examples/uniquecount/src/main/java/org/apache/apex/examples/uniquecount/UniqueKeyValCountExample.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.uniquecount;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.Context;
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.DAG.Locality;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+
+import com.datatorrent.common.partitioner.StatelessPartitioner;
+
+import com.datatorrent.lib.algo.UniqueCounter;
+import com.datatorrent.lib.converter.MapToKeyHashValuePairConverter;
+import com.datatorrent.lib.io.ConsoleOutputOperator;
+import com.datatorrent.lib.util.KeyValPair;
+
+/**
+ * <p>UniqueKeyValCountExample class.</p>
+ *
+ * @since 1.0.2
+ */
+@ApplicationAnnotation(name = "UniqueKeyValueCountExample")
+public class UniqueKeyValCountExample implements StreamingApplication
+{
+
+ @Override
+ public void populateDAG(DAG dag, Configuration entries)
+ {
+ /* Generate random key-value pairs */
+ RandomDataGenerator randGen = dag.addOperator("randomgen", new RandomDataGenerator());
+
+ /* Initialize with three partition to start with */
+ UniqueCounter<KeyValPair<String, Object>> uniqCount =
+ dag.addOperator("uniqevalue", new UniqueCounter<KeyValPair<String, Object>>());
+ MapToKeyHashValuePairConverter<KeyValPair<String, Object>, Integer> converter = dag.addOperator("converter", new MapToKeyHashValuePairConverter());
+ uniqCount.setCumulative(false);
+ dag.setAttribute(randGen, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<UniqueCounter<KeyValPair<String, Object>>>(3));
+
+ ConsoleOutputOperator output = dag.addOperator("output", new ConsoleOutputOperator());
+
+ dag.addStream("datain", randGen.outPort, uniqCount.data);
+ dag.addStream("convert", uniqCount.count, converter.input).setLocality(Locality.THREAD_LOCAL);
+ dag.addStream("consoutput", converter.output, output.input);
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/uniquecount/src/main/java/org/apache/apex/examples/uniquecount/package-info.java
----------------------------------------------------------------------
diff --git a/examples/uniquecount/src/main/java/org/apache/apex/examples/uniquecount/package-info.java b/examples/uniquecount/src/main/java/org/apache/apex/examples/uniquecount/package-info.java
new file mode 100644
index 0000000..713cfb9
--- /dev/null
+++ b/examples/uniquecount/src/main/java/org/apache/apex/examples/uniquecount/package-info.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ Example Application for new Paritionable UniqueCount Operator.
+ */
+package org.apache.apex.examples.uniquecount;
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/uniquecount/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/examples/uniquecount/src/main/resources/META-INF/properties.xml b/examples/uniquecount/src/main/resources/META-INF/properties.xml
new file mode 100644
index 0000000..8742328
--- /dev/null
+++ b/examples/uniquecount/src/main/resources/META-INF/properties.xml
@@ -0,0 +1,29 @@
+<?xml version="1.0"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<configuration>
+ <!--
+ <property>
+ <name>dt.application.{appName}.operator.{opName}.prop.{propName}</name>
+ <value>some-default-value (if value is not specified, it is required from the user or custom config when launching)</value>
+ </property>
+ -->
+</configuration>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/uniquecount/src/test/java/org/apache/apex/examples/uniquecount/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/uniquecount/src/test/java/org/apache/apex/examples/uniquecount/ApplicationTest.java b/examples/uniquecount/src/test/java/org/apache/apex/examples/uniquecount/ApplicationTest.java
new file mode 100644
index 0000000..95242ee
--- /dev/null
+++ b/examples/uniquecount/src/test/java/org/apache/apex/examples/uniquecount/ApplicationTest.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.uniquecount;
+
+import org.junit.Test;
+import org.apache.hadoop.conf.Configuration;
+import com.datatorrent.api.LocalMode;
+
+/**
+ * Test the DAG declaration in local mode.
+ */
+public class ApplicationTest
+{
+ @Test
+ public void testApplication() throws Exception
+ {
+ LocalMode lma = LocalMode.newInstance();
+ new Application().populateDAG(lma.getDAG(), new Configuration(false));
+ LocalMode.Controller lc = lma.getController();
+ lc.run(10000);
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/uniquecount/src/test/java/org/apache/apex/examples/uniquecount/UniqueKeyValExampleTest.java
----------------------------------------------------------------------
diff --git a/examples/uniquecount/src/test/java/org/apache/apex/examples/uniquecount/UniqueKeyValExampleTest.java b/examples/uniquecount/src/test/java/org/apache/apex/examples/uniquecount/UniqueKeyValExampleTest.java
new file mode 100644
index 0000000..6492c9d
--- /dev/null
+++ b/examples/uniquecount/src/test/java/org/apache/apex/examples/uniquecount/UniqueKeyValExampleTest.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.uniquecount;
+
+import org.junit.Test;
+import org.apache.hadoop.conf.Configuration;
+import com.datatorrent.api.LocalMode;
+
+/**
+ * Test the DAG declaration in local mode.
+ */
+public class UniqueKeyValExampleTest
+{
+ @Test
+ public void testApplication() throws Exception
+ {
+ LocalMode lma = LocalMode.newInstance();
+ new UniqueKeyValCountExample().populateDAG(lma.getDAG(), new Configuration(false));
+ LocalMode.Controller lc = lma.getController();
+ lc.run(10000);
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/uniquecount/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/examples/uniquecount/src/test/resources/log4j.properties b/examples/uniquecount/src/test/resources/log4j.properties
new file mode 100644
index 0000000..cf0d19e
--- /dev/null
+++ b/examples/uniquecount/src/test/resources/log4j.properties
@@ -0,0 +1,43 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+log4j.rootLogger=DEBUG,CONSOLE
+
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+log4j.appender.CONSOLE.threshold=${test.log.console.threshold}
+test.log.console.threshold=DEBUG
+
+log4j.appender.RFA=org.apache.log4j.RollingFileAppender
+log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
+log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+log4j.appender.RFA.File=/tmp/app.log
+
+# to enable, add SYSLOG to rootLogger
+log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender
+log4j.appender.SYSLOG.syslogHost=127.0.0.1
+log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout
+log4j.appender.SYSLOG.layout.conversionPattern=${dt.cid} %-5p [%t] %c{2} %x - %m%n
+log4j.appender.SYSLOG.Facility=LOCAL1
+
+log4j.logger.org=info
+#log4j.logger.org.apache.commons.beanutils=warn
+log4j.logger.com.datatorrent=debug
+log4j.logger.org.apache.apex=debug
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/wordcount/pom.xml
----------------------------------------------------------------------
diff --git a/examples/wordcount/pom.xml b/examples/wordcount/pom.xml
new file mode 100644
index 0000000..9106c7c
--- /dev/null
+++ b/examples/wordcount/pom.xml
@@ -0,0 +1,49 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>malhar-examples-wordcount</artifactId>
+ <packaging>jar</packaging>
+
+ <name>Apache Apex Malhar Wordcount Example</name>
+ <description>A very simple application that demonstrates Apex Platform\u2019s streaming window feature.</description>
+
+ <parent>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-examples</artifactId>
+ <version>3.7.0-SNAPSHOT</version>
+ </parent>
+
+ <properties>
+ <skipTests>true</skipTests>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>it.unimi.dsi</groupId>
+ <artifactId>fastutil</artifactId>
+ <version>6.6.4</version>
+ </dependency>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/wordcount/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/examples/wordcount/src/assemble/appPackage.xml b/examples/wordcount/src/assemble/appPackage.xml
new file mode 100644
index 0000000..4138cf2
--- /dev/null
+++ b/examples/wordcount/src/assemble/appPackage.xml
@@ -0,0 +1,59 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+ <id>appPackage</id>
+ <formats>
+ <format>jar</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>${basedir}/target/</directory>
+ <outputDirectory>/app</outputDirectory>
+ <includes>
+ <include>${project.artifactId}-${project.version}.jar</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/target/deps</directory>
+ <outputDirectory>/lib</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/site/conf</directory>
+ <outputDirectory>/conf</outputDirectory>
+ <includes>
+ <include>*.xml</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/META-INF</directory>
+ <outputDirectory>/META-INF</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/app</directory>
+ <outputDirectory>/app</outputDirectory>
+ </fileSet>
+ </fileSets>
+
+</assembly>
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/wordcount/src/main/java/org/apache/apex/examples/wordcount/Application.java
----------------------------------------------------------------------
diff --git a/examples/wordcount/src/main/java/org/apache/apex/examples/wordcount/Application.java b/examples/wordcount/src/main/java/org/apache/apex/examples/wordcount/Application.java
new file mode 100644
index 0000000..6652b79
--- /dev/null
+++ b/examples/wordcount/src/main/java/org/apache/apex/examples/wordcount/Application.java
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.wordcount;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.lib.algo.UniqueCounter;
+import com.datatorrent.lib.io.ConsoleOutputOperator;
+
+/**
+ * Simple Word Count Example : <br>
+ * This is application to count total occurrence of each word from file or any
+ * stream. <br>
+ * <br>
+ *
+ * Functional Description : <br>
+ * This example declares custom input operator to read data file set by user. <br>
+ * This input operator can be replaced by any stream input operator. <br>
+ * <br>
+ *
+ * Custom Attribute(s) : None <br>
+ * <br>
+ *
+ * Input Adapter : <br>
+ * Word input operator opens user specified data file and streams each line to
+ * application. <br>
+ * <br>
+ *
+ * Output Adapter : <br>
+ * Output values are written to console through ConsoleOutputOerator<br>
+ * If needed you can use other output adapters<br>
+ * <br>
+ * <p>
+ * Running Java Test or Main app in IDE:
+ *
+ * <pre>
+ * LocalMode.runApp(new Application(), 600000); // 10 min run
+ * </pre>
+ *
+ * Run Success : <br>
+ * For successful deployment and run, user should see following output on console: <br>
+ * </pre>
+ * {developed=1} {bush\u2019s=2} {roster=1} {council=1} {mankiw=1} {academia=1}
+ * {of=6} {help=1} {are=1} {presidential=1}
+ * </pre> <br> <br>
+ *
+ * Scaling Options : <br>
+ * This operator app can not be scaled, please look at implementation {@link com.datatorrent.lib.algo.UniqueCounterEach} <br> <br>
+ *
+ * Application DAG : <br>
+ * <img src="doc-files/UniqueWordCounter.jpg" width=600px > <br>
+ *
+ * Streaming Window Size : 500ms
+ * Operator Details : <br>
+ * <ul>
+ * <li>
+ * <p><b> The operator wordinput : </b> This operator opens local file, reads each line and sends each word to application.
+ * This can replaced by any input stream by user. <br>
+ * Class : {@link WordCountInputOperator} <br>
+ * Operator Application Window Count : 1 <br>
+ * StateFull : No
+ * </li>
+ * <li>
+ * <p><b> The operator count : </b> This operator aggregates unique key count over one window count(app). <br>
+ * Class : {@link com.datatorrent.lib.algo.UniqueCounterEach} <br>
+ * Operator Application Window Count : 1 <br>
+ * StateFull : No
+ * </li>
+ * <li>
+ * <p><b>The operator Console: </b> This operator just outputs the input tuples to the console (or stdout).
+ * This case it emits unique count of each word over 500ms.
+ * </li>
+ * </ul>
+ *
+ * @since 0.3.2
+ */
+@ApplicationAnnotation(name = "WordCountExample")
+public class Application implements StreamingApplication
+{
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ WordCountInputOperator input = dag.addOperator("wordinput", new WordCountInputOperator());
+ UniqueCounter<String> wordCount = dag.addOperator("count", new UniqueCounter<String>());
+ dag.addStream("wordinput-count", input.outputPort, wordCount.data);
+ ConsoleOutputOperator consoleOperator = dag.addOperator("console", new ConsoleOutputOperator());
+ dag.addStream("count-console",wordCount.count, consoleOperator.input);
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/wordcount/src/main/java/org/apache/apex/examples/wordcount/ApplicationWithQuerySupport.java
----------------------------------------------------------------------
diff --git a/examples/wordcount/src/main/java/org/apache/apex/examples/wordcount/ApplicationWithQuerySupport.java b/examples/wordcount/src/main/java/org/apache/apex/examples/wordcount/ApplicationWithQuerySupport.java
new file mode 100644
index 0000000..699469b
--- /dev/null
+++ b/examples/wordcount/src/main/java/org/apache/apex/examples/wordcount/ApplicationWithQuerySupport.java
@@ -0,0 +1,133 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.wordcount;
+
+import java.net.URI;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.Operator;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+
+import com.datatorrent.lib.appdata.schemas.SchemaUtils;
+import com.datatorrent.lib.appdata.snapshot.AppDataSnapshotServerMap;
+import com.datatorrent.lib.io.ConsoleOutputOperator;
+import com.datatorrent.lib.io.PubSubWebSocketAppDataQuery;
+import com.datatorrent.lib.io.PubSubWebSocketAppDataResult;
+
+/**
+ * Simple example that computes word frequencies from any file dropped into a
+ * monitored directory. It outputs the top N word-frequency pairs for each file
+ * as well globally across all files.
+ * <p>
+ * Each input file generates a corresponding output file in the output directory
+ * containing the top N pairs for that file. The output is also written
+ * to an internal store to support visualization in the UI via queries.
+ * <p>
+ * @since 3.2.0
+ */
+@ApplicationAnnotation(name = "TopNWordsWithQueries")
+public class ApplicationWithQuerySupport implements StreamingApplication
+{
+ private static final Logger LOG = LoggerFactory.getLogger(ApplicationWithQuerySupport.class);
+
+ /**
+ * Name of schema file.
+ */
+ public static final String SNAPSHOT_SCHEMA = "WordDataSchema.json";
+
+ /**
+ * Populates the DAG with operators and connecting streams
+ *
+ * @param dag The directed acyclic graph of operators to populate
+ * @param conf The configuration
+ */
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ // create operators
+ LineReader lineReader = dag.addOperator("lineReader", new LineReader());
+ WordReader wordReader = dag.addOperator("wordReader", new WordReader());
+ WindowWordCount windowWordCount = dag.addOperator("windowWordCount", new WindowWordCount());
+ FileWordCount fileWordCount = dag.addOperator("fileWordCount", new FileWordCount());
+ WordCountWriter wcWriter = dag.addOperator("wcWriter", new WordCountWriter());
+ ConsoleOutputOperator console = dag.addOperator("console", new ConsoleOutputOperator());
+ console.setStringFormat("wordCount: %s");
+
+ // create streams
+
+ dag.addStream("lines", lineReader.output, wordReader.input);
+ dag.addStream("control", lineReader.control, fileWordCount.control);
+ dag.addStream("words", wordReader.output, windowWordCount.input);
+ dag.addStream("windowWordCounts", windowWordCount.output, fileWordCount.input);
+ dag.addStream("fileWordCounts", fileWordCount.fileOutput, wcWriter.input);
+
+ String gatewayAddress = dag.getValue(DAG.GATEWAY_CONNECT_ADDRESS);
+
+ if (!StringUtils.isEmpty(gatewayAddress)) { // add query support
+ URI uri = URI.create("ws://" + gatewayAddress + "/pubsub");
+
+ AppDataSnapshotServerMap snapshotServerFile
+ = dag.addOperator("snapshotServerFile", new AppDataSnapshotServerMap());
+ AppDataSnapshotServerMap snapshotServerGlobal
+ = dag.addOperator("snapshotServerGlobal", new AppDataSnapshotServerMap());
+
+ String snapshotServerJSON = SchemaUtils.jarResourceFileToString(SNAPSHOT_SCHEMA);
+ snapshotServerFile.setSnapshotSchemaJSON(snapshotServerJSON);
+ snapshotServerGlobal.setSnapshotSchemaJSON(snapshotServerJSON);
+
+ PubSubWebSocketAppDataQuery wsQueryFile = new PubSubWebSocketAppDataQuery();
+ PubSubWebSocketAppDataQuery wsQueryGlobal = new PubSubWebSocketAppDataQuery();
+ wsQueryFile.setUri(uri);
+ wsQueryGlobal.setUri(uri);
+
+ snapshotServerFile.setEmbeddableQueryInfoProvider(wsQueryFile);
+ snapshotServerGlobal.setEmbeddableQueryInfoProvider(wsQueryGlobal);
+
+ PubSubWebSocketAppDataResult wsResultFile
+ = dag.addOperator("wsResultFile", new PubSubWebSocketAppDataResult());
+ PubSubWebSocketAppDataResult wsResultGlobal
+ = dag.addOperator("wsResultGlobal", new PubSubWebSocketAppDataResult());
+ wsResultFile.setUri(uri);
+ wsResultGlobal.setUri(uri);
+
+ Operator.InputPort<String> queryResultFilePort = wsResultFile.input;
+ Operator.InputPort<String> queryResultGlobalPort = wsResultGlobal.input;
+
+ dag.addStream("WordCountsFile", fileWordCount.outputPerFile, snapshotServerFile.input, console.input);
+ dag.addStream("WordCountsGlobal", fileWordCount.outputGlobal, snapshotServerGlobal.input);
+
+ dag.addStream("ResultFile", snapshotServerFile.queryResult, queryResultFilePort);
+ dag.addStream("ResultGlobal", snapshotServerGlobal.queryResult, queryResultGlobalPort);
+ } else {
+ //throw new RuntimeException("Error: No GATEWAY_CONNECT_ADDRESS");
+ dag.addStream("WordCounts", fileWordCount.outputPerFile, console.input);
+ }
+
+ LOG.info("done with populateDAG, isDebugEnabled = " + LOG.isDebugEnabled());
+ LOG.info("Returning from populateDAG");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/wordcount/src/main/java/org/apache/apex/examples/wordcount/FileWordCount.java
----------------------------------------------------------------------
diff --git a/examples/wordcount/src/main/java/org/apache/apex/examples/wordcount/FileWordCount.java b/examples/wordcount/src/main/java/org/apache/apex/examples/wordcount/FileWordCount.java
new file mode 100644
index 0000000..a30d0b0
--- /dev/null
+++ b/examples/wordcount/src/main/java/org/apache/apex/examples/wordcount/FileWordCount.java
@@ -0,0 +1,349 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.wordcount;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.datatorrent.api.Context.OperatorContext;
+import com.datatorrent.api.DefaultInputPort;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.annotation.InputPortFieldAnnotation;
+import com.datatorrent.api.annotation.OutputPortFieldAnnotation;
+import com.datatorrent.common.util.BaseOperator;
+
+/**
+ * Computes word frequencies per file and globally, and writes the top N pairs to an output file
+ * and to snapshot servers for visualization.
+ * Currently designed to work with only 1 file at a time; will be enhanced later to support
+ * multiple files dropped into the monitored directory at the same time.
+ *
+ * <p>
+ * Receives per-window list of pairs (word, frequency) on the input port. When the end of a file
+ * is reached, expects to get an EOF on the control port; at the next endWindow, the top N words
+ * and frequencies are computed and emitted to the output ports.
+ * <p>
+ * There are 3 output ports: (a) One for the per-file top N counts emitted when the file is fully
+ * read and is written to the output file. (b) One for the top N counts emitted per window for the
+ * current file to the snapshot server and (c) One for the global top N counts emitted per window
+ * to a different snapshot server.
+ *
+ * Since the EOF is received by a single operator, this operator cannot be partitionable
+ *
+ * @since 3.2.0
+ */
+public class FileWordCount extends BaseOperator
+{
+ private static final Logger LOG = LoggerFactory.getLogger(FileWordCount.class);
+ private static final String GLOBAL = "global";
+
+ /**
+ * If {@literal topN > 0}, only data for the topN most frequent words is output; if topN == 0, the
+ * entire frequency map is output
+ */
+ protected int topN;
+
+ /**
+ * Set to true when an EOF control tuple for the current input file is received; reset to false
+ * when the corresponding output file has been written.
+ */
+ protected boolean eof = false;
+
+ /**
+ * Last component of path (just the file name)
+ * incoming value from control tuple
+ */
+ protected String fileName;
+
+ /**
+ * {@literal (word => frequency)} map: current file, all words
+ */
+ protected Map<String, WCPair> wordMapFile = new HashMap<>();
+
+ /**
+ * {@literal (word => frequency)} map: global, all words
+ */
+ protected Map<String, WCPair> wordMapGlobal = new HashMap<>();
+
+ /**
+ * Singleton list with per file data; sent on {@code outputPerFile}
+ */
+ protected transient List<Map<String, Object>> resultPerFile;
+
+ /**
+ * Singleton list with global data; sent on {@code outputGlobal}
+ */
+ protected transient List<Map<String, Object>> resultGlobal;
+
+ /**
+ * Singleton map of {@code fileName} to sorted list of (word, frequency) pairs
+ */
+ protected transient Map<String, Object> resultFileFinal;
+
+ /**
+ * final list of (word, frequency) pairs written to output file
+ */
+ protected transient List<WCPair> fileFinalList;
+
+ /**
+ * Input port on which per-window {@literal (word => frequency)} map is received; the map
+ * is merged into {@code wordMapFile} and {@code wordMapGlobal}.
+ */
+ public final transient DefaultInputPort<List<WCPair>> input = new DefaultInputPort<List<WCPair>>()
+ {
+ @Override
+ public void process(List<WCPair> list)
+ {
+ // blend incoming list into wordMapFile and wordMapGlobal
+ for (WCPair pair : list) {
+ final String word = pair.word;
+ WCPair filePair = wordMapFile.get(word);
+ if (null != filePair) { // word seen previously in current file
+ WCPair globalPair = wordMapGlobal.get(word); // cannot be null
+ filePair.freq += pair.freq;
+ globalPair.freq += pair.freq;
+ continue;
+ }
+
+ // new word in current file
+ filePair = new WCPair(word, pair.freq);
+ wordMapFile.put(word, filePair);
+
+ // check global map
+ WCPair globalPair = wordMapGlobal.get(word); // may be null
+ if (null != globalPair) { // word seen previously
+ globalPair.freq += pair.freq;
+ continue;
+ }
+
+ // word never seen before
+ globalPair = new WCPair(word, pair.freq);
+ wordMapGlobal.put(word, globalPair);
+ }
+ }
+ };
+
+ /**
+ * Control port on which the current file name is received to indicate EOF
+ */
+ @InputPortFieldAnnotation(optional = true)
+ public final transient DefaultInputPort<String> control = new DefaultInputPort<String>()
+ {
+ @Override
+ public void process(String msg)
+ {
+ if (msg.isEmpty()) { // sanity check
+ throw new RuntimeException("Empty file path");
+ }
+ LOG.info("FileWordCount: EOF for {}, topN = {}", msg, topN);
+ fileName = msg;
+ eof = true;
+ // NOTE: current version only supports processing one file at a time.
+ }
+ };
+
+ /**
+ * Output port for current file output
+ */
+ public final transient DefaultOutputPort<List<Map<String, Object>>>
+ outputPerFile = new DefaultOutputPort<>();
+
+ /**
+ * Output port for global output
+ */
+ @OutputPortFieldAnnotation(optional = true)
+ public final transient DefaultOutputPort<List<Map<String, Object>>>
+ outputGlobal = new DefaultOutputPort<>();
+
+ /**
+ * Tuple is singleton map {@code fileName => TopNMap} where {@code TopNMap} is the final
+ * top N pairs for current file and will be written to the output file; emitted in the
+ * {@code endWindow()} call after an EOF
+ */
+ public final transient DefaultOutputPort<Map<String, Object>>
+ fileOutput = new DefaultOutputPort<>();
+
+ /**
+ * Get the number of top (word, frequency) pairs that will be output
+ */
+ public int getTopN()
+ {
+ return topN;
+ }
+
+ /**
+ * Set the number of top (word, frequency) pairs that will be output
+ * @param n The new number
+ */
+ public void setTopN(int n)
+ {
+ topN = n;
+ }
+
+ /**
+ * {@inheritDoc}
+ * Initialize various map and list fields
+ */
+ @Override
+ public void setup(OperatorContext context)
+ {
+ if (null == wordMapFile) {
+ wordMapFile = new HashMap<>();
+ }
+ if (null == wordMapGlobal) {
+ wordMapGlobal = new HashMap<>();
+ }
+ resultPerFile = new ArrayList(1);
+ resultGlobal = new ArrayList(1);
+ // singleton map {<fileName> => fileFinalList}; cannot populate it yet since we need fileName
+ resultFileFinal = new HashMap<>(1);
+ fileFinalList = new ArrayList<>();
+ }
+
+ /**
+ * {@inheritDoc}
+ * This is where we do most of the work:
+ * 1. Sort global map and emit top N pairs
+ * 2. Sort current file map and emit top N pairs
+ * 3. If we've seen EOF, emit top N pairs on port connected to file writer and clear all per-file
+ * data structures.
+ */
+ @Override
+ public void endWindow()
+ {
+ LOG.info("FileWordCount: endWindow for {}, topN = {}", fileName, topN);
+
+ if (wordMapFile.isEmpty()) { // no words found
+ if (eof) { // write empty list to fileOutput port
+ // got EOF, so output empty list to output file
+ fileFinalList.clear();
+ resultFileFinal.put(fileName, fileFinalList);
+ fileOutput.emit(resultFileFinal);
+
+ // reset for next file
+ eof = false;
+ fileName = null;
+ resultFileFinal.clear();
+ }
+ LOG.info("FileWordCount: endWindow for {}, no words, topN = {}", fileName, topN);
+ return;
+ }
+
+ LOG.info("FileWordCount: endWindow for {}, wordMapFile.size = {}, topN = {}", fileName, wordMapFile.size(), topN);
+
+ // get topN list for this file and, if we have EOF, emit to fileOutput port
+
+ // get topN global list and emit to global output port
+ getTopNMap(wordMapGlobal, resultGlobal);
+ LOG.info("FileWordCount: resultGlobal.size = {}", resultGlobal.size());
+ outputGlobal.emit(resultGlobal);
+
+ // get topN list for this file and emit to file output port
+ getTopNMap(wordMapFile, resultPerFile);
+ LOG.info("FileWordCount: resultPerFile.size = {}", resultPerFile.size());
+ outputPerFile.emit(resultPerFile);
+
+ if (eof) { // got EOF earlier
+ if (null == fileName) { // need file name to emit topN pairs to file writer
+ throw new RuntimeException("EOF but no fileName at endWindow");
+ }
+
+ // so compute final topN list from wordMapFile into fileFinalList and emit it
+ getTopNList(wordMapFile);
+ resultFileFinal.put(fileName, fileFinalList);
+ fileOutput.emit(resultFileFinal);
+
+ // reset for next file
+ eof = false;
+ fileName = null;
+ wordMapFile.clear();
+ resultFileFinal.clear();
+ }
+ }
+
+ /**
+ * Get topN frequencies from map, convert each pair to a singleton map and append to result
+ * This map is suitable input to AppDataSnapshotServer
+ * MUST have {@code map.size() > 0} here
+ */
+ private void getTopNMap(final Map<String, WCPair> map, List<Map<String, Object>> result)
+ {
+ final ArrayList<WCPair> list = new ArrayList<>(map.values());
+
+ // sort entries in descending order of frequency
+ Collections.sort(list, new Comparator<WCPair>()
+ {
+ @Override
+ public int compare(WCPair o1, WCPair o2)
+ {
+ return (int)(o2.freq - o1.freq);
+ }
+ });
+
+ if (topN > 0) {
+ list.subList(topN, map.size()).clear(); // retain only the first topN entries
+ }
+
+ // convert each pair (word, freq) of list to a map with 2 elements
+ // {("word": <word>, "count": freq)} and append to list
+ //
+ result.clear();
+ for (WCPair pair : list) {
+ Map<String, Object> wmap = new HashMap<>(2);
+ wmap.put("word", pair.word);
+ wmap.put("count", pair.freq);
+ result.add(wmap);
+ }
+ LOG.info("FileWordCount:getTopNMap: result.size = {}", result.size());
+ list.clear();
+ }
+
+ /**
+ * Populate fileFinalList with topN frequencies from argument
+ * This list is suitable input to WordCountWriter which writes it to a file
+ * MUST have {@code map.size() > 0} here
+ */
+ private void getTopNList(final Map<String, WCPair> map)
+ {
+ fileFinalList.clear();
+ fileFinalList.addAll(map.values());
+
+ // sort entries in descending order of frequency
+ Collections.sort(fileFinalList, new Comparator<WCPair>()
+ {
+ @Override
+ public int compare(WCPair o1, WCPair o2)
+ {
+ return (int)(o2.freq - o1.freq);
+ }
+ });
+
+ if (topN > 0) {
+ fileFinalList.subList(topN, map.size()).clear(); // retain only the first topN entries
+ }
+ LOG.info("FileWordCount:getTopNList: fileFinalList.size = {}", fileFinalList.size());
+ }
+}
[26/30] apex-malhar git commit: Renamed demos to examples. Packages
and artifactid names are changed as suggested.
Posted by th...@apache.org.
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/cookbook/CombinePerKeyExamples.java
----------------------------------------------------------------------
diff --git a/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/cookbook/CombinePerKeyExamples.java b/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/cookbook/CombinePerKeyExamples.java
deleted file mode 100644
index bfdb268..0000000
--- a/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/cookbook/CombinePerKeyExamples.java
+++ /dev/null
@@ -1,285 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.apex.malhar.stream.sample.cookbook;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.apex.malhar.lib.window.TriggerOption;
-import org.apache.apex.malhar.lib.window.Tuple;
-import org.apache.apex.malhar.lib.window.WindowOption;
-import org.apache.apex.malhar.lib.window.accumulation.ReduceFn;
-import org.apache.apex.malhar.stream.api.ApexStream;
-import org.apache.apex.malhar.stream.api.CompositeStreamTransform;
-import org.apache.apex.malhar.stream.api.WindowedStream;
-import org.apache.apex.malhar.stream.api.function.Function;
-import org.apache.apex.malhar.stream.api.impl.StreamFactory;
-
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.Context;
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.DefaultInputPort;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.api.InputOperator;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-import com.datatorrent.common.util.BaseOperator;
-import com.datatorrent.lib.util.KeyValPair;
-
-import static org.apache.apex.malhar.stream.api.Option.Options.name;
-
-/**
- * An example that reads the public 'Shakespeare' data, and for each word in
- * the dataset that is over a given length, generates a string containing the
- * list of play names in which that word appears
- *
- * <p>Concepts: the combine transform, which lets you combine the values in a
- * key-grouped Collection
- *
- *
- * @since 3.5.0
- */
-@ApplicationAnnotation(name = "CombinePerKeyExamples")
-public class CombinePerKeyExamples implements StreamingApplication
-{
- // Use the shakespeare public BigQuery sample
- private static final String SHAKESPEARE_TABLE = "publicdata:samples.shakespeare";
- // We'll track words >= this word length across all plays in the table.
- private static final int MIN_WORD_LENGTH = 0;
-
- /**
- * Examines each row in the input table. If the word is greater than or equal to MIN_WORD_LENGTH,
- * outputs word, play_name.
- */
- static class ExtractLargeWordsFn implements Function.MapFunction<SampleBean, KeyValPair<String, String>>
- {
-
- @Override
- public KeyValPair<String, String> f(SampleBean input)
- {
- String playName = input.getCorpus();
- String word = input.getWord();
- if (word.length() >= MIN_WORD_LENGTH) {
- return new KeyValPair<>(word, playName);
- } else {
- return null;
- }
- }
- }
-
-
- /**
- * Prepares the output data which is in same bean
- */
- static class FormatShakespeareOutputFn implements Function.MapFunction<Tuple.WindowedTuple<KeyValPair<String, String>>, SampleBean>
- {
- @Override
- public SampleBean f(Tuple.WindowedTuple<KeyValPair<String, String>> input)
- {
- return new SampleBean(input.getValue().getKey(), input.getValue().getValue());
- }
- }
-
- /**
- * A reduce function to concat two strings together.
- */
- public static class Concat extends ReduceFn<String>
- {
- @Override
- public String reduce(String input1, String input2)
- {
- return input1 + ", " + input2;
- }
- }
-
- /**
- * Reads the public 'Shakespeare' data, and for each word in the dataset
- * over a given length, generates a string containing the list of play names
- * in which that word appears.
- */
- private static class PlaysForWord extends CompositeStreamTransform<ApexStream<SampleBean>, WindowedStream<SampleBean>>
- {
-
- @Override
- public WindowedStream<SampleBean> compose(ApexStream<SampleBean> inputStream)
- {
- return inputStream
- // Extract words from the input SampleBeam stream.
- .map(new ExtractLargeWordsFn(), name("ExtractLargeWordsFn"))
-
- // Apply window and trigger option to the streams.
- .window(new WindowOption.GlobalWindow(), new TriggerOption().accumulatingFiredPanes().withEarlyFiringsAtEvery(1))
-
- // Apply reduceByKey transformation to concat the names of all the plays that a word has appeared in together.
- .reduceByKey(new Concat(), new Function.ToKeyValue<KeyValPair<String,String>, String, String>()
- {
- @Override
- public Tuple<KeyValPair<String, String>> f(KeyValPair<String, String> input)
- {
- return new Tuple.PlainTuple<KeyValPair<String, String>>(input);
- }
- }, name("Concat"))
-
- // Format the output back to a SampleBeam object.
- .map(new FormatShakespeareOutputFn(), name("FormatShakespeareOutputFn"));
- }
- }
-
-
- /**
- * A Java Beam class that contains information about a word appears in a corpus written by Shakespeare.
- */
- public static class SampleBean
- {
-
- public SampleBean()
- {
-
- }
-
- public SampleBean(String word, String corpus)
- {
- this.word = word;
- this.corpus = corpus;
- }
-
- @Override
- public String toString()
- {
- return this.word + " : " + this.corpus;
- }
-
- private String word;
-
- private String corpus;
-
- public void setWord(String word)
- {
- this.word = word;
- }
-
- public String getWord()
- {
- return word;
- }
-
- public void setCorpus(String corpus)
- {
- this.corpus = corpus;
- }
-
- public String getCorpus()
- {
- return corpus;
- }
- }
-
- /**
- * A dummy info generator to generate {@link SampleBean} objects to mimic reading from real 'Shakespeare'
- * data.
- */
- public static class SampleInput extends BaseOperator implements InputOperator
- {
-
- public final transient DefaultOutputPort<SampleBean> beanOutput = new DefaultOutputPort();
- private String[] words = new String[]{"A", "B", "C", "D", "E", "F", "G"};
- private String[] corpuses = new String[]{"1", "2", "3", "4", "5", "6", "7", "8"};
- private static int i;
-
- @Override
- public void setup(Context.OperatorContext context)
- {
- super.setup(context);
- i = 0;
- }
-
- @Override
- public void emitTuples()
- {
- while (i < 1) {
- for (String word : words) {
- for (String corpus : corpuses) {
- try {
- Thread.sleep(50);
- beanOutput.emit(new SampleBean(word, corpus));
- } catch (Exception e) {
- // Ignore it
- }
- }
- }
- i++;
- }
-
- }
- }
-
- public static class Collector extends BaseOperator
- {
- private static List<SampleBean> result;
- private static boolean done = false;
-
- public static List<SampleBean> getResult()
- {
- return result;
- }
-
- public static boolean isDone()
- {
- return done;
- }
-
- @Override
- public void setup(Context.OperatorContext context)
- {
- result = new ArrayList<>();
- done = false;
- }
-
- public final transient DefaultInputPort<SampleBean> input = new DefaultInputPort<SampleBean>()
- {
- @Override
- public void process(SampleBean tuple)
- {
- if (tuple.getWord().equals("F")) {
- done = true;
- }
- result.add(tuple);
- }
- };
- }
-
- /**
- * Populate dag using High-Level API.
- * @param dag
- * @param conf
- */
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- SampleInput input = new SampleInput();
- Collector collector = new Collector();
- StreamFactory.fromInput(input, input.beanOutput, name("input"))
- .addCompositeStreams(new PlaysForWord())
- .print(name("console"))
- .endWith(collector, collector.input, name("Collector"))
- .populateDag(dag);
-
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/cookbook/DeDupExample.java
----------------------------------------------------------------------
diff --git a/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/cookbook/DeDupExample.java b/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/cookbook/DeDupExample.java
deleted file mode 100644
index 4df5fe7..0000000
--- a/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/cookbook/DeDupExample.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.apex.malhar.stream.sample.cookbook;
-
-import java.util.Arrays;
-import java.util.List;
-
-import org.joda.time.Duration;
-
-import org.apache.apex.malhar.lib.window.TriggerOption;
-import org.apache.apex.malhar.lib.window.Tuple;
-import org.apache.apex.malhar.lib.window.WindowOption;
-import org.apache.apex.malhar.lib.window.accumulation.RemoveDuplicates;
-import org.apache.apex.malhar.stream.api.ApexStream;
-import org.apache.apex.malhar.stream.api.function.Function;
-import org.apache.apex.malhar.stream.api.impl.StreamFactory;
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.Context;
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.DefaultInputPort;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-import com.datatorrent.common.util.BaseOperator;
-
-import static org.apache.apex.malhar.stream.api.Option.Options.name;
-
-/**
- * Beam DeDupExample.
- *
- * @since 3.5.0
- */
-@ApplicationAnnotation(name = "DeDupExample")
-public class DeDupExample implements StreamingApplication
-{
-
- public static class Collector extends BaseOperator
- {
- private static Tuple.WindowedTuple<List<String>> result;
- private static boolean done = false;
-
- public static Tuple.WindowedTuple<List<String>> getResult()
- {
- return result;
- }
-
- public static boolean isDone()
- {
- return done;
- }
-
- @Override
- public void setup(Context.OperatorContext context)
- {
- super.setup(context);
- result = new Tuple.WindowedTuple<>();
- done = false;
- }
-
- public transient DefaultInputPort<Tuple.WindowedTuple<List<String>>> input = new DefaultInputPort<Tuple.WindowedTuple<List<String>>>()
- {
- @Override
- public void process(Tuple.WindowedTuple<List<String>> tuple)
- {
- result = tuple;
- if (result.getValue().contains("bye")) {
- done = true;
- }
- }
- };
- }
-
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- Collector collector = new Collector();
-
- // Create a stream that reads from files in a local folder and output lines one by one to downstream.
- ApexStream<String> stream = StreamFactory.fromFolder("./src/test/resources/wordcount", name("textInput"))
-
- // Extract all the words from the input line of text.
- .flatMap(new Function.FlatMapFunction<String, String>()
- {
- @Override
- public Iterable<String> f(String input)
- {
- return Arrays.asList(input.split("[\\p{Punct}\\s]+"));
- }
- }, name("ExtractWords"))
-
- // Change the words to lower case, also shutdown the app when the word "bye" is detected.
- .map(new Function.MapFunction<String, String>()
- {
- @Override
- public String f(String input)
- {
- return input.toLowerCase();
- }
- }, name("ToLowerCase"));
-
- // Apply window and trigger option.
- stream.window(new WindowOption.GlobalWindow(),
- new TriggerOption().accumulatingFiredPanes().withEarlyFiringsAtEvery(Duration.standardSeconds(1)))
-
- // Remove the duplicate words and print out the result.
- .accumulate(new RemoveDuplicates<String>(), name("RemoveDuplicates"))
- .print(name("console"))
- .endWith(collector, collector.input)
- .populateDag(dag);
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/cookbook/InputPojo.java
----------------------------------------------------------------------
diff --git a/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/cookbook/InputPojo.java b/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/cookbook/InputPojo.java
deleted file mode 100644
index 834964c..0000000
--- a/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/cookbook/InputPojo.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.apex.malhar.stream.sample.cookbook;
-
-/**
- * Tuple class for JDBC input of {@link MaxPerKeyExamples}.
- *
- * @since 3.5.0
- */
-public class InputPojo extends Object
-{
- private int month;
- private int day;
- private int year;
- private double meanTemp;
-
- @Override
- public String toString()
- {
- return "PojoEvent [month=" + getMonth() + ", day=" + getDay() + ", year=" + getYear() + ", meanTemp=" + getMeanTemp() + "]";
- }
-
- public void setMonth(int month)
- {
- this.month = month;
- }
-
- public int getMonth()
- {
- return this.month;
- }
-
- public void setDay(int day)
- {
- this.day = day;
- }
-
- public int getDay()
- {
- return day;
- }
-
- public void setYear(int year)
- {
- this.year = year;
- }
-
- public int getYear()
- {
- return year;
- }
-
- public void setMeanTemp(double meanTemp)
- {
- this.meanTemp = meanTemp;
- }
-
- public double getMeanTemp()
- {
- return meanTemp;
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/cookbook/MaxPerKeyExamples.java
----------------------------------------------------------------------
diff --git a/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/cookbook/MaxPerKeyExamples.java b/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/cookbook/MaxPerKeyExamples.java
deleted file mode 100644
index 9fd9495..0000000
--- a/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/cookbook/MaxPerKeyExamples.java
+++ /dev/null
@@ -1,205 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.apex.malhar.stream.sample.cookbook;
-
-import java.util.List;
-
-import org.apache.apex.malhar.lib.window.TriggerOption;
-import org.apache.apex.malhar.lib.window.Tuple;
-import org.apache.apex.malhar.lib.window.Window;
-import org.apache.apex.malhar.lib.window.WindowOption;
-import org.apache.apex.malhar.lib.window.accumulation.Max;
-import org.apache.apex.malhar.stream.api.ApexStream;
-import org.apache.apex.malhar.stream.api.CompositeStreamTransform;
-import org.apache.apex.malhar.stream.api.WindowedStream;
-import org.apache.apex.malhar.stream.api.function.Function;
-import org.apache.apex.malhar.stream.api.impl.StreamFactory;
-import org.apache.hadoop.conf.Configuration;
-
-import static java.sql.Types.DOUBLE;
-import static java.sql.Types.INTEGER;
-
-import com.google.common.collect.Lists;
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-import com.datatorrent.lib.db.jdbc.JdbcFieldInfo;
-import com.datatorrent.lib.db.jdbc.JdbcPOJOInputOperator;
-import com.datatorrent.lib.db.jdbc.JdbcPOJOInsertOutputOperator;
-import com.datatorrent.lib.db.jdbc.JdbcStore;
-import com.datatorrent.lib.db.jdbc.JdbcTransactionalStore;
-import com.datatorrent.lib.util.FieldInfo;
-import com.datatorrent.lib.util.KeyValPair;
-
-import static org.apache.apex.malhar.stream.api.Option.Options.name;
-
-/**
- * MaxPerKeyExamples Application from Beam
- *
- * @since 3.5.0
- */
-@ApplicationAnnotation(name = "MaxPerKeyExamples")
-public class MaxPerKeyExamples implements StreamingApplication
-{
-
- /**
- * A map function to extract the mean temperature from {@link InputPojo}.
- */
- public static class ExtractTempFn implements Function.MapFunction<InputPojo, KeyValPair<Integer, Double>>
- {
- @Override
- public KeyValPair<Integer, Double> f(InputPojo row)
- {
- Integer month = row.getMonth();
- Double meanTemp = row.getMeanTemp();
- return new KeyValPair<Integer, Double>(month, meanTemp);
- }
- }
-
-
- /**
- * A map function to format output to {@link OutputPojo}.
- */
- public static class FormatMaxesFn implements Function.MapFunction<Tuple.WindowedTuple<KeyValPair<Integer, Double>>, OutputPojo>
- {
- @Override
- public OutputPojo f(Tuple.WindowedTuple<KeyValPair<Integer, Double>> input)
- {
- OutputPojo row = new OutputPojo();
- row.setMonth(input.getValue().getKey());
- row.setMeanTemp(input.getValue().getValue());
- return row;
- }
- }
-
- /**
- * A composite transformation to perform three tasks:
- * 1. extract the month and its mean temperature from input pojo.
- * 2. find the maximum mean temperature for every month.
- * 3. format the result to a output pojo object.
- */
- public static class MaxMeanTemp extends CompositeStreamTransform<WindowedStream<InputPojo>, WindowedStream<OutputPojo>>
- {
- @Override
- public WindowedStream<OutputPojo> compose(WindowedStream<InputPojo> rows)
- {
- // InputPojo... => <month, meanTemp> ...
- WindowedStream<KeyValPair<Integer, Double>> temps = rows.map(new ExtractTempFn(), name("ExtractTempFn"));
-
- // month, meanTemp... => <month, max mean temp>...
- WindowedStream<Tuple.WindowedTuple<KeyValPair<Integer, Double>>> tempMaxes =
- temps.accumulateByKey(new Max<Double>(),
- new Function.ToKeyValue<KeyValPair<Integer, Double>, Integer, Double>()
- {
- @Override
- public Tuple<KeyValPair<Integer, Double>> f(KeyValPair<Integer, Double> input)
- {
- return new Tuple.WindowedTuple<KeyValPair<Integer, Double>>(Window.GlobalWindow.INSTANCE, input);
- }
- }, name("MaxPerMonth"));
-
- // <month, max>... => OutputPojo...
- WindowedStream<OutputPojo> results = tempMaxes.map(new FormatMaxesFn(), name("FormatMaxesFn"));
-
- return results;
- }
- }
-
- /**
- * Method to set field info for {@link JdbcPOJOInputOperator}.
- * @return
- */
- private List<FieldInfo> addInputFieldInfos()
- {
- List<FieldInfo> fieldInfos = Lists.newArrayList();
- fieldInfos.add(new FieldInfo("MONTH", "month", FieldInfo.SupportType.INTEGER));
- fieldInfos.add(new FieldInfo("DAY", "day", FieldInfo.SupportType.INTEGER));
- fieldInfos.add(new FieldInfo("YEAR", "year", FieldInfo.SupportType.INTEGER));
- fieldInfos.add(new FieldInfo("MEANTEMP", "meanTemp", FieldInfo.SupportType.DOUBLE));
- return fieldInfos;
- }
-
- /**
- * Method to set field info for {@link JdbcPOJOInsertOutputOperator}.
- * @return
- */
- private List<JdbcFieldInfo> addOutputFieldInfos()
- {
- List<JdbcFieldInfo> fieldInfos = Lists.newArrayList();
- fieldInfos.add(new JdbcFieldInfo("MONTH", "month", JdbcFieldInfo.SupportType.INTEGER, INTEGER));
- fieldInfos.add(new JdbcFieldInfo("MEANTEMP", "meanTemp", JdbcFieldInfo.SupportType.DOUBLE, DOUBLE));
- return fieldInfos;
- }
-
-
- /**
- * Populate the dag using High-Level API.
- * @param dag
- * @param conf
- */
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- JdbcPOJOInputOperator jdbcInput = new JdbcPOJOInputOperator();
- jdbcInput.setFieldInfos(addInputFieldInfos());
-
- JdbcStore store = new JdbcStore();
- jdbcInput.setStore(store);
-
- JdbcPOJOInsertOutputOperator jdbcOutput = new JdbcPOJOInsertOutputOperator();
- jdbcOutput.setFieldInfos(addOutputFieldInfos());
- JdbcTransactionalStore outputStore = new JdbcTransactionalStore();
- jdbcOutput.setStore(outputStore);
-
- // Create stream that reads from a Jdbc Input.
- ApexStream<Object> stream = StreamFactory.fromInput(jdbcInput, jdbcInput.outputPort, name("jdbcInput"))
-
- // Apply window and trigger option to the stream.
- .window(new WindowOption.GlobalWindow(), new TriggerOption().accumulatingFiredPanes().withEarlyFiringsAtEvery(1))
-
- // Because Jdbc Input sends out a stream of Object, need to cast them to InputPojo.
- .map(new Function.MapFunction<Object, InputPojo>()
- {
- @Override
- public InputPojo f(Object input)
- {
- return (InputPojo)input;
- }
- }, name("ObjectToInputPojo"))
-
- // Plug in the composite transformation to the stream to calculate the maximum temperature for each month.
- .addCompositeStreams(new MaxMeanTemp())
-
- // Cast the resulted OutputPojo to Object for Jdbc Output to consume.
- .map(new Function.MapFunction<OutputPojo, Object>()
- {
- @Override
- public Object f(OutputPojo input)
- {
- return (Object)input;
- }
- }, name("OutputPojoToObject"))
-
- // Output the result to Jdbc Output.
- .endWith(jdbcOutput, jdbcOutput.input, name("jdbcOutput"));
-
- stream.populateDag(dag);
-
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/cookbook/OutputPojo.java
----------------------------------------------------------------------
diff --git a/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/cookbook/OutputPojo.java b/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/cookbook/OutputPojo.java
deleted file mode 100644
index f3d0c64..0000000
--- a/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/cookbook/OutputPojo.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.apex.malhar.stream.sample.cookbook;
-
-/**
- * OutputPojo Tuple Class for jdbcOutput of {@link MaxPerKeyExamples}.
- *
- * @since 3.5.0
- */
-public class OutputPojo
-{
- private int month;
- private double meanTemp;
-
- @Override
- public String toString()
- {
- return "PojoEvent [month=" + getMonth() + ", meanTemp=" + getMeanTemp() + "]";
- }
-
- public void setMonth(int month)
- {
- this.month = month;
- }
-
- public int getMonth()
- {
- return this.month;
- }
-
- public void setMeanTemp(double meanTemp)
- {
- this.meanTemp = meanTemp;
- }
-
- public double getMeanTemp()
- {
- return meanTemp;
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/cookbook/TriggerExample.java
----------------------------------------------------------------------
diff --git a/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/cookbook/TriggerExample.java b/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/cookbook/TriggerExample.java
deleted file mode 100644
index 962faa5..0000000
--- a/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/cookbook/TriggerExample.java
+++ /dev/null
@@ -1,577 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.apex.malhar.stream.sample.cookbook;
-
-import java.util.Date;
-import java.util.Objects;
-
-import org.joda.time.Duration;
-
-import org.apache.apex.malhar.lib.window.TriggerOption;
-import org.apache.apex.malhar.lib.window.Tuple;
-import org.apache.apex.malhar.lib.window.WindowOption;
-import org.apache.apex.malhar.stream.api.ApexStream;
-import org.apache.apex.malhar.stream.api.CompositeStreamTransform;
-import org.apache.apex.malhar.stream.api.WindowedStream;
-import org.apache.apex.malhar.stream.api.function.Function;
-import org.apache.apex.malhar.stream.api.impl.StreamFactory;
-
-import com.datatorrent.lib.util.KeyValPair;
-
-/**
- * This example illustrates the basic concepts behind triggering. It shows how to use different
- * trigger definitions to produce partial (speculative) results before all the data is processed and
- * to control when updated results are produced for late data. The example performs a streaming
- * analysis of the data coming in from PubSub and writes the results to BigQuery. It divides the
- * data into {@link Window windows} to be processed, and demonstrates using various kinds of
- * {@link org.apache.beam.sdk.transforms.windowing.Trigger triggers} to control when the results for
- * each window are emitted.
- *
- * <p> This example uses a portion of real traffic data from San Diego freeways. It contains
- * readings from sensor stations set up along each freeway. Each sensor reading includes a
- * calculation of the 'total flow' across all lanes in that freeway direction.
- *
- * <p> Concepts:
- * <pre>
- * 1. The default triggering behavior
- * 2. Late data with the default trigger
- * 3. How to get speculative estimates
- * 4. Combining late data and speculative estimates
- * </pre>
- *
- * <p> Before running this example, it will be useful to familiarize yourself with Dataflow triggers
- * and understand the concept of 'late data',
- * See: <a href="https://cloud.google.com/dataflow/model/triggers">
- * https://cloud.google.com/dataflow/model/triggers </a> and
- * <a href="https://cloud.google.com/dataflow/model/windowing#Advanced">
- * https://cloud.google.com/dataflow/model/windowing#Advanced </a>
- *
- * <p> The example pipeline reads data from a Pub/Sub topic. By default, running the example will
- * also run an auxiliary pipeline to inject data from the default {@code --input} file to the
- * {@code --pubsubTopic}. The auxiliary pipeline puts a timestamp on the injected data so that the
- * example pipeline can operate on <i>event time</i> (rather than arrival time). The auxiliary
- * pipeline also randomly simulates late data, by setting the timestamps of some of the data
- * elements to be in the past. You may override the default {@code --input} with the file of your
- * choosing or set {@code --input=""} which will disable the automatic Pub/Sub injection, and allow
- * you to use a separate tool to publish to the given topic.
- *
- * <p> The example is configured to use the default Pub/Sub topic and the default BigQuery table
- * from the example common package (there are no defaults for a general Dataflow pipeline).
- * You can override them by using the {@code --pubsubTopic}, {@code --bigQueryDataset}, and
- * {@code --bigQueryTable} options. If the Pub/Sub topic or the BigQuery table do not exist,
- * the example will try to create them.
- *
- * <p> The pipeline outputs its results to a BigQuery table.
- * Here are some queries you can use to see interesting results:
- * Replace {@code <enter_table_name>} in the query below with the name of the BigQuery table.
- * Replace {@code <enter_window_interval>} in the query below with the window interval.
- *
- * <p> To see the results of the default trigger,
- * Note: When you start up your pipeline, you'll initially see results from 'late' data. Wait after
- * the window duration, until the first pane of non-late data has been emitted, to see more
- * interesting results.
- * {@code SELECT * FROM enter_table_name WHERE triggerType = "default" ORDER BY window DESC}
- *
- * <p> To see the late data i.e. dropped by the default trigger,
- * {@code SELECT * FROM <enter_table_name> WHERE triggerType = "withAllowedLateness" and
- * (timing = "LATE" or timing = "ON_TIME") and freeway = "5" ORDER BY window DESC, processingTime}
- *
- * <p>To see the the difference between accumulation mode and discarding mode,
- * {@code SELECT * FROM <enter_table_name> WHERE (timing = "LATE" or timing = "ON_TIME") AND
- * (triggerType = "withAllowedLateness" or triggerType = "sequential") and freeway = "5" ORDER BY
- * window DESC, processingTime}
- *
- * <p> To see speculative results every minute,
- * {@code SELECT * FROM <enter_table_name> WHERE triggerType = "speculative" and freeway = "5"
- * ORDER BY window DESC, processingTime}
- *
- * <p> To see speculative results every five minutes after the end of the window
- * {@code SELECT * FROM <enter_table_name> WHERE triggerType = "sequential" and timing != "EARLY"
- * and freeway = "5" ORDER BY window DESC, processingTime}
- *
- * <p> To see the first and the last pane for a freeway in a window for all the trigger types,
- * {@code SELECT * FROM <enter_table_name> WHERE (isFirst = true or isLast = true) ORDER BY window}
- *
- * <p> To reduce the number of results for each query we can add additional where clauses.
- * For examples, To see the results of the default trigger,
- * {@code SELECT * FROM <enter_table_name> WHERE triggerType = "default" AND freeway = "5" AND
- * window = "<enter_window_interval>"}
- *
- * <p> The example will try to cancel the pipelines on the signal to terminate the process (CTRL-C)
- * and then exits.
- *
- * @since 3.5.0
- */
-
-public class TriggerExample
-{
- //Numeric value of fixed window duration, in minutes
- public static final int WINDOW_DURATION = 30;
- // Constants used in triggers.
- // Speeding up ONE_MINUTE or FIVE_MINUTES helps you get an early approximation of results.
- // ONE_MINUTE is used only with processing time before the end of the window
- public static final Duration ONE_MINUTE = Duration.standardMinutes(1);
- // FIVE_MINUTES is used only with processing time after the end of the window
- public static final Duration FIVE_MINUTES = Duration.standardMinutes(5);
- // ONE_DAY is used to specify the amount of lateness allowed for the data elements.
- public static final Duration ONE_DAY = Duration.standardDays(1);
-
- /**
- * This transform demonstrates using triggers to control when data is produced for each window
- * Consider an example to understand the results generated by each type of trigger.
- * The example uses "freeway" as the key. Event time is the timestamp associated with the data
- * element and processing time is the time when the data element gets processed in the pipeline.
- * For freeway 5, suppose there are 10 elements in the [10:00:00, 10:30:00) window.
- * Key (freeway) | Value (totalFlow) | event time | processing time
- * 5 | 50 | 10:00:03 | 10:00:47
- * 5 | 30 | 10:01:00 | 10:01:03
- * 5 | 30 | 10:02:00 | 11:07:00
- * 5 | 20 | 10:04:10 | 10:05:15
- * 5 | 60 | 10:05:00 | 11:03:00
- * 5 | 20 | 10:05:01 | 11.07:30
- * 5 | 60 | 10:15:00 | 10:27:15
- * 5 | 40 | 10:26:40 | 10:26:43
- * 5 | 60 | 10:27:20 | 10:27:25
- * 5 | 60 | 10:29:00 | 11:11:00
- *
- * <p> Dataflow tracks a watermark which records up to what point in event time the data is
- * complete. For the purposes of the example, we'll assume the watermark is approximately 15m
- * behind the current processing time. In practice, the actual value would vary over time based
- * on the systems knowledge of the current PubSub delay and contents of the backlog (data
- * that has not yet been processed).
- *
- * <p> If the watermark is 15m behind, then the window [10:00:00, 10:30:00) (in event time) would
- * close at 10:44:59, when the watermark passes 10:30:00.
- */
- static class CalculateTotalFlow
- extends CompositeStreamTransform<ApexStream<String>, WindowedStream<SampleBean>>
- {
- private int windowDuration;
-
- CalculateTotalFlow(int windowDuration)
- {
- this.windowDuration = windowDuration;
- }
-
- @Override
- public WindowedStream<SampleBean> compose(ApexStream<String> inputStream)
- {
- // Concept #1: The default triggering behavior
- // By default Dataflow uses a trigger which fires when the watermark has passed the end of the
- // window. This would be written {@code Repeatedly.forever(AfterWatermark.pastEndOfWindow())}.
-
- // The system also defaults to dropping late data -- data which arrives after the watermark
- // has passed the event timestamp of the arriving element. This means that the default trigger
- // will only fire once.
-
- // Each pane produced by the default trigger with no allowed lateness will be the first and
- // last pane in the window, and will be ON_TIME.
-
- // The results for the example above with the default trigger and zero allowed lateness
- // would be:
- // Key (freeway) | Value (totalFlow) | numberOfRecords | isFirst | isLast | timing
- // 5 | 260 | 6 | true | true | ON_TIME
-
- // At 11:03:00 (processing time) the system watermark may have advanced to 10:54:00. As a
- // result, when the data record with event time 10:05:00 arrives at 11:03:00, it is considered
- // late, and dropped.
-
- WindowedStream<SampleBean> defaultTriggerResults = inputStream
- .window(new WindowOption.TimeWindows(Duration.standardMinutes(windowDuration)),
- new TriggerOption().discardingFiredPanes())
- .addCompositeStreams(new TotalFlow("default"));
-
- // Concept #2: Late data with the default trigger
- // This uses the same trigger as concept #1, but allows data that is up to ONE_DAY late. This
- // leads to each window staying open for ONE_DAY after the watermark has passed the end of the
- // window. Any late data will result in an additional pane being fired for that same window.
-
- // The first pane produced will be ON_TIME and the remaining panes will be LATE.
- // To definitely get the last pane when the window closes, use
- // .withAllowedLateness(ONE_DAY, ClosingBehavior.FIRE_ALWAYS).
-
- // The results for the example above with the default trigger and ONE_DAY allowed lateness
- // would be:
- // Key (freeway) | Value (totalFlow) | numberOfRecords | isFirst | isLast | timing
- // 5 | 260 | 6 | true | false | ON_TIME
- // 5 | 60 | 1 | false | false | LATE
- // 5 | 30 | 1 | false | false | LATE
- // 5 | 20 | 1 | false | false | LATE
- // 5 | 60 | 1 | false | false | LATE
- WindowedStream<SampleBean> withAllowedLatenessResults = inputStream
- .window(new WindowOption.TimeWindows(Duration.standardMinutes(windowDuration)),
- new TriggerOption().discardingFiredPanes(),
- Duration.standardDays(1))
- .addCompositeStreams(new TotalFlow("withAllowedLateness"));
-
- // Concept #3: How to get speculative estimates
- // We can specify a trigger that fires independent of the watermark, for instance after
- // ONE_MINUTE of processing time. This allows us to produce speculative estimates before
- // all the data is available. Since we don't have any triggers that depend on the watermark
- // we don't get an ON_TIME firing. Instead, all panes are either EARLY or LATE.
-
- // We also use accumulatingFiredPanes to build up the results across each pane firing.
-
- // The results for the example above for this trigger would be:
- // Key (freeway) | Value (totalFlow) | numberOfRecords | isFirst | isLast | timing
- // 5 | 80 | 2 | true | false | EARLY
- // 5 | 100 | 3 | false | false | EARLY
- // 5 | 260 | 6 | false | false | EARLY
- // 5 | 320 | 7 | false | false | LATE
- // 5 | 370 | 9 | false | false | LATE
- // 5 | 430 | 10 | false | false | LATE
-
- ApexStream<SampleBean> speculativeResults = inputStream
- .window(new WindowOption.TimeWindows(Duration.standardMinutes(windowDuration)),
- //Trigger fires every minute
- new TriggerOption().withEarlyFiringsAtEvery(Duration.standardMinutes(1))
- // After emitting each pane, it will continue accumulating the elements so that each
- // approximation includes all of the previous data in addition to the newly arrived
- // data.
- .accumulatingFiredPanes(),
- Duration.standardDays(1))
- .addCompositeStreams(new TotalFlow("speculative"));
-
- // Concept #4: Combining late data and speculative estimates
- // We can put the previous concepts together to get EARLY estimates, an ON_TIME result,
- // and LATE updates based on late data.
-
- // Each time a triggering condition is satisfied it advances to the next trigger.
- // If there are new elements this trigger emits a window under following condition:
- // > Early approximations every minute till the end of the window.
- // > An on-time firing when the watermark has passed the end of the window
- // > Every five minutes of late data.
-
- // Every pane produced will either be EARLY, ON_TIME or LATE.
-
- // The results for the example above for this trigger would be:
- // Key (freeway) | Value (totalFlow) | numberOfRecords | isFirst | isLast | timing
- // 5 | 80 | 2 | true | false | EARLY
- // 5 | 100 | 3 | false | false | EARLY
- // 5 | 260 | 6 | false | false | EARLY
- // [First pane fired after the end of the window]
- // 5 | 320 | 7 | false | false | ON_TIME
- // 5 | 430 | 10 | false | false | LATE
-
- // For more possibilities of how to build advanced triggers, see {@link Trigger}.
- WindowedStream<SampleBean> sequentialResults = inputStream
- .window(new WindowOption.TimeWindows(Duration.standardMinutes(windowDuration)),
- // Speculative every ONE_MINUTE
- new TriggerOption().withEarlyFiringsAtEvery(Duration.standardMinutes(1))
- .withLateFiringsAtEvery(Duration.standardMinutes(5))
- // After emitting each pane, it will continue accumulating the elements so that each
- // approximation includes all of the previous data in addition to the newly arrived
- // data.
- .accumulatingFiredPanes(),
- Duration.standardDays(1))
- .addCompositeStreams(new TotalFlow("sequential"));
-
- return sequentialResults;
- }
-
- }
-
- //////////////////////////////////////////////////////////////////////////////////////////////////
- // The remaining parts of the pipeline are needed to produce the output for each
- // concept above. Not directly relevant to understanding the trigger examples.
-
- /**
- * Calculate total flow and number of records for each freeway and format the results to TableRow
- * objects, to save to BigQuery.
- */
- static class TotalFlow extends
- CompositeStreamTransform<WindowedStream<String>, WindowedStream<SampleBean>>
- {
- private String triggerType;
-
- public TotalFlow(String triggerType)
- {
- this.triggerType = triggerType;
- }
-
- @Override
- public WindowedStream<SampleBean> compose(WindowedStream<String> inputStream)
- {
-
- WindowedStream<KeyValPair<String, Iterable<Integer>>> flowPerFreeway = inputStream
- .groupByKey(new ExtractFlowInfo());
-
- return flowPerFreeway
- .map(new Function.MapFunction<KeyValPair<String, Iterable<Integer>>, KeyValPair<String, String>>()
- {
- @Override
- public KeyValPair<String, String> f(KeyValPair<String, Iterable<Integer>> input)
- {
- Iterable<Integer> flows = input.getValue();
- Integer sum = 0;
- Long numberOfRecords = 0L;
- for (Integer value : flows) {
- sum += value;
- numberOfRecords++;
- }
- return new KeyValPair<>(input.getKey(), sum + "," + numberOfRecords);
- }
- })
- .map(new FormatTotalFlow(triggerType));
- }
- }
-
- /**
- * Format the results of the Total flow calculation to a TableRow, to save to BigQuery.
- * Adds the triggerType, pane information, processing time and the window timestamp.
- */
- static class FormatTotalFlow implements Function.MapFunction<KeyValPair<String, String>, SampleBean>
- {
- private String triggerType;
-
- public FormatTotalFlow(String triggerType)
- {
- this.triggerType = triggerType;
- }
-
- @Override
- public SampleBean f(KeyValPair<String, String> input)
- {
- String[] values = input.getValue().split(",");
- //TODO need to have a callback to get the metadata like window id, pane id, timestamps etc.
- return new SampleBean(triggerType, input.getKey(), Integer.parseInt(values[0]), Long
- .parseLong(values[1]), null, false, false, null, null, new Date());
- }
- }
-
- public static class SampleBean
- {
- public SampleBean()
- {
- }
-
- private String triggerType;
-
- private String freeway;
-
- private int totalFlow;
-
- private long numberOfRecords;
-
- private String window;
-
- private boolean isFirst;
-
- private boolean isLast;
-
- private Date timing;
-
- private Date eventTime;
-
- private Date processingTime;
-
- @Override
- public boolean equals(Object o)
- {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- SampleBean that = (SampleBean)o;
- return totalFlow == that.totalFlow &&
- numberOfRecords == that.numberOfRecords &&
- isFirst == that.isFirst &&
- isLast == that.isLast &&
- Objects.equals(triggerType, that.triggerType) &&
- Objects.equals(freeway, that.freeway) &&
- Objects.equals(window, that.window) &&
- Objects.equals(timing, that.timing) &&
- Objects.equals(eventTime, that.eventTime) &&
- Objects.equals(processingTime, that.processingTime);
- }
-
- @Override
- public int hashCode()
- {
- return Objects
- .hash(triggerType, freeway, totalFlow, numberOfRecords, window, isFirst, isLast, timing, eventTime,
- processingTime);
- }
-
- public SampleBean(String triggerType, String freeway, int totalFlow, long numberOfRecords, String window, boolean isFirst, boolean isLast, Date timing, Date eventTime, Date processingTime)
- {
-
- this.triggerType = triggerType;
- this.freeway = freeway;
- this.totalFlow = totalFlow;
- this.numberOfRecords = numberOfRecords;
- this.window = window;
- this.isFirst = isFirst;
- this.isLast = isLast;
- this.timing = timing;
- this.eventTime = eventTime;
- this.processingTime = processingTime;
- }
-
- public String getTriggerType()
- {
- return triggerType;
- }
-
- public void setTriggerType(String triggerType)
- {
- this.triggerType = triggerType;
- }
-
- public String getFreeway()
- {
- return freeway;
- }
-
- public void setFreeway(String freeway)
- {
- this.freeway = freeway;
- }
-
- public int getTotalFlow()
- {
- return totalFlow;
- }
-
- public void setTotalFlow(int totalFlow)
- {
- this.totalFlow = totalFlow;
- }
-
- public long getNumberOfRecords()
- {
- return numberOfRecords;
- }
-
- public void setNumberOfRecords(long numberOfRecords)
- {
- this.numberOfRecords = numberOfRecords;
- }
-
- public String getWindow()
- {
- return window;
- }
-
- public void setWindow(String window)
- {
- this.window = window;
- }
-
- public boolean isFirst()
- {
- return isFirst;
- }
-
- public void setFirst(boolean first)
- {
- isFirst = first;
- }
-
- public boolean isLast()
- {
- return isLast;
- }
-
- public void setLast(boolean last)
- {
- isLast = last;
- }
-
- public Date getTiming()
- {
- return timing;
- }
-
- public void setTiming(Date timing)
- {
- this.timing = timing;
- }
-
- public Date getEventTime()
- {
- return eventTime;
- }
-
- public void setEventTime(Date eventTime)
- {
- this.eventTime = eventTime;
- }
-
- public Date getProcessingTime()
- {
- return processingTime;
- }
-
- public void setProcessingTime(Date processingTime)
- {
- this.processingTime = processingTime;
- }
- }
-
- /**
- * Extract the freeway and total flow in a reading.
- * Freeway is used as key since we are calculating the total flow for each freeway.
- */
- static class ExtractFlowInfo implements Function.ToKeyValue<String, String, Integer>
- {
- @Override
- public Tuple<KeyValPair<String, Integer>> f(String input)
- {
- String[] laneInfo = input.split(",");
- if (laneInfo[0].equals("timestamp")) {
- // Header row
- return null;
- }
- if (laneInfo.length < 48) {
- //Skip the invalid input.
- return null;
- }
- String freeway = laneInfo[2];
- Integer totalFlow = tryIntegerParse(laneInfo[7]);
- // Ignore the records with total flow 0 to easily understand the working of triggers.
- // Skip the records with total flow -1 since they are invalid input.
- if (totalFlow == null || totalFlow <= 0) {
- return null;
- }
- return new Tuple.PlainTuple<>(new KeyValPair<>(freeway, totalFlow));
- }
- }
-
- private static final String PUBSUB_TIMESTAMP_LABEL_KEY = "timestamp_ms";
-
- public static void main(String[] args) throws Exception
- {
- StreamFactory.fromFolder("some folder")
- .addCompositeStreams(new CalculateTotalFlow(60));
-
- }
-
- private static Integer tryIntegerParse(String number)
- {
- try {
- return Integer.parseInt(number);
- } catch (NumberFormatException e) {
- return null;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/highlevelapi/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/demos/highlevelapi/src/main/resources/META-INF/properties.xml b/demos/highlevelapi/src/main/resources/META-INF/properties.xml
deleted file mode 100644
index ead0460..0000000
--- a/demos/highlevelapi/src/main/resources/META-INF/properties.xml
+++ /dev/null
@@ -1,141 +0,0 @@
-<?xml version="1.0"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<configuration>
- <!--
- <property>
- <name>dt.application.{appName}.operator.{opName}.prop.{propName}</name>
- <value>some-default-value (if value is not specified, it is required from the user or custom config when launching)</value>
- </property>
- -->
-
- <!-- Properties for TwitterAutoComplete, please fill out all of them to make the application work -->
- <property>
- <name>dt.application.TwitterAutoComplete.operator.tweetSampler.consumerKey</name>
- <value></value>
- </property>
- <property>
- <name>dt.application.TwitterAutoComplete.operator.tweetSampler.consumerSecret</name>
- <value></value>
- </property>
- <property>
- <name>dt.application.TwitterAutoComplete.operator.tweetSampler.accessToken</name>
- <value></value>
- </property>
- <property>
- <name>dt.application.TwitterAutoComplete.operator.tweetSampler.accessTokenSecret</name>
- <value></value>
- </property>
-
- <!-- Properties for StreamingWordExtract -->
- <property>
- <name>dt.application.StreamingWordExtract.operator.jdbcOutput.prop.store.userName</name>
- <value>root</value>
- </property>
- <property>
- <name>dt.application.StreamingWordExtract.operator.jdbcOutput.prop.store.password</name>
- <value>password</value>
- </property>
- <property>
- <name>dt.application.StreamingWordExtract.operator.jdbcOutput.prop.store.databaseDriver</name>
- <value>org.hsqldb.jdbcDriver</value>
- </property>
- <property>
- <name>dt.application.StreamingWordExtract.operator.jdbcOutput.prop.batchSize</name>
- <value>5</value>
- </property>
- <property>
- <name>dt.application.StreamingWordExtract.operator.jdbcOutput.port.input.attr.TUPLE_CLASS</name>
- <value>org.apache.apex.malhar.stream.sample.complete.PojoEvent</value>
- </property>
- <property>
- <name>dt.application.StreamingWordExtract.operator.jdbcOutput.prop.store.databaseUrl</name>
- <value>jdbc:hsqldb:mem:test</value>
- </property>
- <property>
- <name>dt.application.StreamingWordExtract.operator.jdbcOutput.prop.tablename</name>
- <value>Test</value>
- </property>
-
- <!-- Properties for MaxPerKeyExamples -->
- <property>
- <name>dt.application.MaxPerKeyExamples.operator.jdbcInput.prop.store.userName</name>
- <value>root</value>
- </property>
- <property>
- <name>dt.application.MaxPerKeyExamples.operator.jdbcInput.prop.store.password</name>
- <value>password</value>
- </property>
- <property>
- <name>dt.application.MaxPerKeyExamples.operator.jdbcInput.prop.store.databaseDriver</name>
- <value>org.hsqldb.jdbcDriver</value>
- </property>
- <property>
- <name>dt.application.MaxPerKeyExamples.operator.jdbcInput.prop.batchSize</name>
- <value>5</value>
- </property>
- <property>
- <name>dt.application.MaxPerKeyExamples.operator.jdbcInput.port.outputPort.attr.TUPLE_CLASS</name>
- <value>org.apache.apex.malhar.stream.sample.cookbook.InputPojo</value>
- </property>
- <property>
- <name>dt.application.MaxPerKeyExamples.operator.jdbcInput.prop.store.databaseUrl</name>
- <value>jdbc:hsqldb:mem:test</value>
- </property>
- <property>
- <name>dt.application.MaxPerKeyExamples.operator.jdbcInput.prop.tableName</name>
- <value>InputTable</value>
- </property>
- <property>
- <name>dt.application.MaxPerKeyExamples.operator.jdbcInput.prop.query</name>
- <value>SELECT * FROM InputTable;</value>
- </property>
- <property>
- <name>dt.application.MaxPerKeyExamples.operator.jdbcOutput.prop.store.userName</name>
- <value>root</value>
- </property>
- <property>
- <name>dt.application.MaxPerKeyExamples.operator.jdbcOutput.prop.store.password</name>
- <value>password</value>
- </property>
- <property>
- <name>dt.application.MaxPerKeyExamples.operator.jdbcOutput.prop.store.databaseDriver</name>
- <value>org.hsqldb.jdbcDriver</value>
- </property>
- <property>
- <name>dt.application.MaxPerKeyExamples.operator.jdbcOutput.prop.batchSize</name>
- <value>5</value>
- </property>
- <property>
- <name>dt.application.MaxPerKeyExamples.operator.jdbcOutput.port.input.attr.TUPLE_CLASS</name>
- <value>org.apache.apex.malhar.stream.sample.cookbook.OutputPojo</value>
- </property>
- <property>
- <name>dt.application.MaxPerKeyExamples.operator.jdbcOutput.prop.store.databaseUrl</name>
- <value>jdbc:hsqldb:mem:test</value>
- </property>
- <property>
- <name>dt.application.MaxPerKeyExamples.operator.jdbcOutput.prop.tablename</name>
- <value>OutputTable</value>
- </property>
-
-</configuration>
-
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/MinimalWordCountTest.java
----------------------------------------------------------------------
diff --git a/demos/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/MinimalWordCountTest.java b/demos/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/MinimalWordCountTest.java
deleted file mode 100644
index c078683..0000000
--- a/demos/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/MinimalWordCountTest.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.apex.malhar.stream.sample;
-
-import java.util.concurrent.Callable;
-
-import org.junit.Assert;
-import org.junit.Test;
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.LocalMode;
-import com.datatorrent.stram.StramLocalCluster;
-
-/**
- * Test for {@link MinimalWordCount}.
- */
-public class MinimalWordCountTest
-{
- @Test
- public void MinimalWordCountTest() throws Exception
- {
- LocalMode lma = LocalMode.newInstance();
- Configuration conf = new Configuration(false);
- conf.set("dt.application.MinimalWordCount.operator.console.silent", "true");
- MinimalWordCount app = new MinimalWordCount();
-
- lma.prepareDAG(app, conf);
-
- LocalMode.Controller lc = lma.getController();
- ((StramLocalCluster)lc).setExitCondition(new Callable<Boolean>()
- {
- @Override
- public Boolean call() throws Exception
- {
- return MinimalWordCount.Collector.isDone();
- }
- });
-
- lc.run(10000);
-
- Assert.assertTrue(MinimalWordCount.Collector.result.get("error") == 7);
- Assert.assertTrue(MinimalWordCount.Collector.result.get("word") == 119);
- Assert.assertTrue(MinimalWordCount.Collector.result.get("bye") == 1);
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/WindowedWordCountTest.java
----------------------------------------------------------------------
diff --git a/demos/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/WindowedWordCountTest.java b/demos/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/WindowedWordCountTest.java
deleted file mode 100644
index f0c51f6..0000000
--- a/demos/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/WindowedWordCountTest.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.apex.malhar.stream.sample;
-
-import java.util.Map;
-import java.util.concurrent.Callable;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.LocalMode;
-import com.datatorrent.lib.util.KeyValPair;
-import com.datatorrent.stram.StramLocalCluster;
-
-/**
- * Testing the TwitterAutoComplete Application. In order to run this test, you need to create an app
- * at https://apps.twitter.com, then generate your consumer and access keys and tokens, and set the following properties
- * for the application before running it:
- * Your application consumer key,
- * Your application consumer secret,
- * Your twitter access token, and
- * Your twitter access token secret.
- */
-public class WindowedWordCountTest
-{
- @Test
- public void WindowedWordCountTest() throws Exception
- {
- LocalMode lma = LocalMode.newInstance();
- Configuration conf = new Configuration(false);
- conf.set("dt.application.WindowedWordCount.operator.console.silent", "true");
- lma.prepareDAG(new WindowedWordCount(), conf);
- LocalMode.Controller lc = lma.getController();
- ((StramLocalCluster)lc).setExitCondition(new Callable<Boolean>()
- {
- @Override
- public Boolean call() throws Exception
- {
- return WindowedWordCount.Collector.isDone();
- }
- });
-
- lc.run(60000);
-
- Assert.assertEquals(127, countSum(WindowedWordCount.Collector.getResult()));
- Assert.assertEquals(28, countSumWord(WindowedWordCount.Collector.getResult(), "word2"));
- Assert.assertEquals(7, countSumWord(WindowedWordCount.Collector.getResult(), "error"));
- Assert.assertEquals(21, countSumWord(WindowedWordCount.Collector.getResult(), "word9"));
- Assert.assertEquals(1, countSumWord(WindowedWordCount.Collector.getResult(), "bye"));
- }
-
- public long countSum(Map<KeyValPair<Long, String>, Long> map)
- {
- long sum = 0;
- for (long count : map.values()) {
- sum += count;
- }
- return sum;
- }
-
- public long countSumWord(Map<KeyValPair<Long, String>, Long> map, String word)
- {
- long sum = 0;
- for (Map.Entry<KeyValPair<Long, String>, Long> entry : map.entrySet()) {
- if (entry.getKey().getValue().equals(word)) {
- sum += entry.getValue();
- }
- }
- return sum;
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/complete/AutoCompleteTest.java
----------------------------------------------------------------------
diff --git a/demos/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/complete/AutoCompleteTest.java b/demos/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/complete/AutoCompleteTest.java
deleted file mode 100644
index 4ed2d5d..0000000
--- a/demos/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/complete/AutoCompleteTest.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.apex.malhar.stream.sample.complete;
-
-import java.util.concurrent.Callable;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.LocalMode;
-import com.datatorrent.stram.StramLocalCluster;
-
-/**
- * Testing the AutoComplete Application
- */
-public class AutoCompleteTest
-{
-
- @Test
- public void AutoCompleteTest() throws Exception
- {
- LocalMode lma = LocalMode.newInstance();
- Configuration conf = new Configuration(false);
- conf.set("dt.application.AutoComplete.operator.console.silent", "true");
- lma.prepareDAG(new AutoComplete(), conf);
- LocalMode.Controller lc = lma.getController();
-
- ((StramLocalCluster)lc).setExitCondition(new Callable<Boolean>()
- {
- @Override
- public Boolean call() throws Exception
- {
- return AutoComplete.Collector.isDone();
- }
- });
-
- lc.run(200000);
-
- Assert.assertTrue(AutoComplete.Collector.getResult().containsKey("had"));
- Assert.assertTrue(AutoComplete.Collector.getResult().containsKey("hadoop"));
- Assert.assertEquals(2, AutoComplete.Collector.getResult().get("mapreduce").get(0).getCount());
-
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/complete/StreamingWordExtractTest.java
----------------------------------------------------------------------
diff --git a/demos/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/complete/StreamingWordExtractTest.java b/demos/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/complete/StreamingWordExtractTest.java
deleted file mode 100644
index dc9cdec..0000000
--- a/demos/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/complete/StreamingWordExtractTest.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.apex.malhar.stream.sample.complete;
-
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.concurrent.Callable;
-
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import org.apache.hadoop.conf.Configuration;
-
-import com.google.common.base.Throwables;
-import com.datatorrent.api.LocalMode;
-import com.datatorrent.lib.db.jdbc.JdbcTransactionalStore;
-import com.datatorrent.stram.StramLocalCluster;
-
-/**
- * Testing StreamingWordExtract application
- */
-public class StreamingWordExtractTest
-{
- private static final String TUPLE_CLASS = "org.apache.apex.malhar.stream.sample.complete.PojoEvent";
- private static final String DB_DRIVER = "org.h2.Driver";
- private static final String DB_URL = "jdbc:h2:~/test";
- private static final String TABLE_NAME = "Test";
- private static final String USER_NAME = "root";
- private static final String PSW = "password";
-
- @BeforeClass
- public static void setup()
- {
- try {
- Class.forName(DB_DRIVER).newInstance();
-
- Connection con = DriverManager.getConnection(DB_URL,USER_NAME,PSW);
- Statement stmt = con.createStatement();
-
- String createMetaTable = "CREATE TABLE IF NOT EXISTS " + JdbcTransactionalStore.DEFAULT_META_TABLE + " ( "
- + JdbcTransactionalStore.DEFAULT_APP_ID_COL + " VARCHAR(100) NOT NULL, "
- + JdbcTransactionalStore.DEFAULT_OPERATOR_ID_COL + " INT NOT NULL, "
- + JdbcTransactionalStore.DEFAULT_WINDOW_COL + " BIGINT NOT NULL, "
- + "UNIQUE (" + JdbcTransactionalStore.DEFAULT_APP_ID_COL + ", "
- + JdbcTransactionalStore.DEFAULT_OPERATOR_ID_COL + ", " + JdbcTransactionalStore.DEFAULT_WINDOW_COL + ") "
- + ")";
- stmt.executeUpdate(createMetaTable);
-
- String createTable = "CREATE TABLE IF NOT EXISTS " + TABLE_NAME
- + "(STRINGVALUE VARCHAR(255))";
- stmt.executeUpdate(createTable);
-
- } catch (Throwable e) {
- throw Throwables.propagate(e);
- }
- }
-
- @After
- public void cleanTable()
- {
- try {
- Connection con = DriverManager.getConnection(DB_URL,USER_NAME,PSW);
- Statement stmt = con.createStatement();
- String dropTable = "drop table " + TABLE_NAME;
- stmt.executeUpdate(dropTable);
- } catch (SQLException e) {
- throw new RuntimeException(e);
- }
- }
-
- public void setConfig(Configuration conf)
- {
- conf.set("dt.operator.jdbcOutput.prop.store.userName", USER_NAME);
- conf.set("dt.operator.jdbcOutput.prop.store.password", PSW);
- conf.set("dt.operator.jdbcOutput.prop.store.databaseDriver", DB_DRIVER);
- conf.set("dt.operator.jdbcOutput.prop.batchSize", "5");
- conf.set("dt.operator.jdbcOutput.port.input.attr.TUPLE_CLASS", TUPLE_CLASS);
- conf.set("dt.operator.jdbcOutput.prop.store.databaseUrl", DB_URL);
- conf.set("dt.operator.jdbcOutput.prop.tablename", TABLE_NAME);
- }
-
- public int getNumOfEventsInStore()
- {
- Connection con;
- try {
- con = DriverManager.getConnection(DB_URL,USER_NAME,PSW);
- Statement stmt = con.createStatement();
-
- String countQuery = "SELECT count(*) from " + TABLE_NAME;
- ResultSet resultSet = stmt.executeQuery(countQuery);
- resultSet.next();
- return resultSet.getInt(1);
- } catch (SQLException e) {
- throw new RuntimeException("fetching count", e);
- }
- }
-
- @Test
- public void StreamingWordExtractTest() throws Exception
- {
- LocalMode lma = LocalMode.newInstance();
- Configuration conf = new Configuration(false);
- setConfig(conf);
- StreamingWordExtract app = new StreamingWordExtract();
- lma.prepareDAG(app, conf);
- LocalMode.Controller lc = lma.getController();
-
- ((StramLocalCluster)lc).setExitCondition(new Callable<Boolean>()
- {
- @Override
- public Boolean call() throws Exception
- {
- return getNumOfEventsInStore() == 36;
- }
- });
-
- lc.run(10000);
-
- Assert.assertEquals(app.getWordCount(), getNumOfEventsInStore());
- Assert.assertEquals(app.getEntriesMapped(), getNumOfEventsInStore());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/complete/TopWikipediaSessionsTest.java
----------------------------------------------------------------------
diff --git a/demos/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/complete/TopWikipediaSessionsTest.java b/demos/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/complete/TopWikipediaSessionsTest.java
deleted file mode 100644
index fddf511..0000000
--- a/demos/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/complete/TopWikipediaSessionsTest.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.apex.malhar.stream.sample.complete;
-
-import java.util.List;
-import java.util.concurrent.Callable;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.LocalMode;
-import com.datatorrent.stram.StramLocalCluster;
-
-/**
- * Testing the {@link TopWikipediaSessions} Application.
- */
-public class TopWikipediaSessionsTest
-{
- @Test
- public void TopWikipediaSessionsTest() throws Exception
- {
- LocalMode lma = LocalMode.newInstance();
- Configuration conf = new Configuration(false);
- conf.set("dt.application.TopWikipediaSessions.operator.console.silent", "true");
- lma.prepareDAG(new TopWikipediaSessions(), conf);
- LocalMode.Controller lc = lma.getController();
-
- ((StramLocalCluster)lc).setExitCondition(new Callable<Boolean>()
- {
- @Override
- public Boolean call() throws Exception
- {
- return TopWikipediaSessions.SessionGen.getTupleCount() >= 250;
- }
- });
-
- lc.run(30000);
-
- for (int i = 0; i < TopWikipediaSessions.Collector.getResult().size(); i++) {
- Assert.assertTrue(isInOrder(TopWikipediaSessions.Collector.getResult().get(i)));
- }
- }
-
- public boolean isInOrder(List<TopWikipediaSessions.TempWrapper> input)
- {
- if (input.size() == 0 || input.size() == 1) {
- return true;
- }
- for (int i = 0; i < input.size() - 2; i++) {
- if (input.get(i).getValue().getValue() < input.get(i + 1).getValue().getValue()) {
- return false;
- }
- }
- return true;
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/complete/TrafficRoutesTest.java
----------------------------------------------------------------------
diff --git a/demos/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/complete/TrafficRoutesTest.java b/demos/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/complete/TrafficRoutesTest.java
deleted file mode 100644
index 766fa60..0000000
--- a/demos/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/complete/TrafficRoutesTest.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.apex.malhar.stream.sample.complete;
-
-import java.util.Map;
-import java.util.concurrent.Callable;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.LocalMode;
-import com.datatorrent.lib.util.KeyValPair;
-import com.datatorrent.stram.StramLocalCluster;
-
-/**
- * Testing the {@link TrafficRoutes} Application.
- */
-public class TrafficRoutesTest
-{
-
- @Test
- public void TrafficRoutesTest() throws Exception
- {
- LocalMode lma = LocalMode.newInstance();
- Configuration conf = new Configuration(false);
- conf.set("dt.application.TrafficRoutes.operator.console.silent", "true");
- lma.prepareDAG(new TrafficRoutes(), conf);
- LocalMode.Controller lc = lma.getController();
-
- ((StramLocalCluster)lc).setExitCondition(new Callable<Boolean>()
- {
- @Override
- public Boolean call() throws Exception
- {
- return TrafficRoutes.InfoGen.getTupleCount() >= 100;
- }
- });
-
- lc.run(60000);
-
- Assert.assertTrue(!TrafficRoutes.Collector.getResult().isEmpty());
- for (Map.Entry<KeyValPair<Long, String>, KeyValPair<Double, Boolean>> entry : TrafficRoutes.Collector.getResult().entrySet()) {
- Assert.assertTrue(entry.getValue().getKey() <= 75);
- Assert.assertTrue(entry.getValue().getKey() >= 55);
- Assert.assertTrue(entry.getKey().getValue().equals("SDRoute1") || entry.getKey().getValue().equals("SDRoute2"));
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/complete/TwitterAutoCompleteTest.java
----------------------------------------------------------------------
diff --git a/demos/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/complete/TwitterAutoCompleteTest.java b/demos/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/complete/TwitterAutoCompleteTest.java
deleted file mode 100644
index 9ba2f25..0000000
--- a/demos/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/complete/TwitterAutoCompleteTest.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.apex.malhar.stream.sample.complete;
-
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.LocalMode;
-
-/**
- * Testing the TwitterAutoComplete Application. In order to run this test, you need to create an app
- * at https://apps.twitter.com, then generate your consumer and access keys and tokens, and set the following properties
- * for the application before running it:
- * Your application consumer key,
- * Your application consumer secret,
- * Your twitter access token, and
- * Your twitter access token secret.
- *
- * This test is mainly for local demonstration purpose. Default time to run the application is 1 minute, please
- * set the time you need to run the application before you run.
- */
-public class TwitterAutoCompleteTest
-{
- private static final Logger logger = LoggerFactory.getLogger(org.apache.apex.malhar.stream.sample.complete.AutoCompleteTest.class);
-
- @Test
- @Ignore
- public void TwitterAutoCompleteTest() throws Exception
- {
- LocalMode lma = LocalMode.newInstance();
- Configuration conf = new Configuration(false);
- //uncomment the following lines and change YOUR_XXX to the corresponding information needed.
- //conf.set("dt.application.TwitterAutoComplete.operator.tweetSampler.consumerKey", "YOUR_CONSUMERKEY");
- //conf.set("dt.application.TwitterAutoComplete.operator.tweetSampler.consumerSecret", "YOUR_CONSUERSECRET");
- //conf.set("dt.application.TwitterAutoComplete.operator.tweetSampler.accessToken", "YOUR_ACCESSTOKEN");
- //conf.set("dt.application.TwitterAutoComplete.operator.tweetSampler.accessTokenSecret", "YOUR_TOKENSECRET");
- lma.prepareDAG(new TwitterAutoComplete(), conf);
- LocalMode.Controller lc = lma.getController();
- long start = System.currentTimeMillis();
- lc.run(60000); // Set your desired time to run the application here.
- long end = System.currentTimeMillis();
- long time = end - start;
- logger.info("Test used " + time + " ms");
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/cookbook/CombinePerKeyExamplesTest.java
----------------------------------------------------------------------
diff --git a/demos/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/cookbook/CombinePerKeyExamplesTest.java b/demos/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/cookbook/CombinePerKeyExamplesTest.java
deleted file mode 100644
index 1e14fff..0000000
--- a/demos/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/cookbook/CombinePerKeyExamplesTest.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.apex.malhar.stream.sample.cookbook;
-
-import java.util.concurrent.Callable;
-import org.junit.Assert;
-import org.junit.Test;
-import org.apache.hadoop.conf.Configuration;
-import com.datatorrent.api.LocalMode;
-import com.datatorrent.stram.StramLocalCluster;
-
-/**
- * Test for {@link CombinePerKeyExamples}.
- */
-public class CombinePerKeyExamplesTest
-{
- @Test
- public void CombinePerKeyExamplesTest() throws Exception
- {
- LocalMode lma = LocalMode.newInstance();
- Configuration conf = new Configuration(false);
- conf.set("dt.application.CombinePerKeyExamples.operator.console.silent", "true");
- CombinePerKeyExamples app = new CombinePerKeyExamples();
-
- lma.prepareDAG(app, conf);
-
- LocalMode.Controller lc = lma.getController();
- ((StramLocalCluster)lc).setExitCondition(new Callable<Boolean>()
- {
- @Override
- public Boolean call() throws Exception
- {
- return CombinePerKeyExamples.Collector.isDone();
- }
- });
- lc.run(100000);
-
- Assert.assertTrue(CombinePerKeyExamples.Collector.getResult().get(CombinePerKeyExamples.Collector.getResult().size() - 2).getCorpus().contains("1, 2, 3, 4, 5, 6, 7, 8"));
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/cookbook/DeDupExampleTest.java
----------------------------------------------------------------------
diff --git a/demos/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/cookbook/DeDupExampleTest.java b/demos/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/cookbook/DeDupExampleTest.java
deleted file mode 100644
index 7f93f50..0000000
--- a/demos/highlevelapi/src/test/java/org/apache/apex/malhar/stream/sample/cookbook/DeDupExampleTest.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.apex.malhar.stream.sample.cookbook;
-
-import java.util.concurrent.Callable;
-
-import org.junit.Assert;
-import org.junit.Test;
-import org.apache.hadoop.conf.Configuration;
-import com.datatorrent.api.LocalMode;
-
-import com.datatorrent.stram.StramLocalCluster;
-
-
-/**
- * Test for {@link DeDupExample}.
- */
-public class DeDupExampleTest
-{
- @Test
- public void DeDupExampleTest() throws Exception
- {
- LocalMode lma = LocalMode.newInstance();
- Configuration conf = new Configuration(false);
- conf.set("dt.application.DeDupExample.operator.console.silent", "true");
- DeDupExample app = new DeDupExample();
- lma.prepareDAG(app, conf);
- LocalMode.Controller lc = lma.getController();
- ((StramLocalCluster)lc).setExitCondition(new Callable<Boolean>()
- {
- @Override
- public Boolean call() throws Exception
- {
- return DeDupExample.Collector.isDone();
- }
- });
- lc.run(50000);
-
- Assert.assertEquals(9, DeDupExample.Collector.getResult().getValue().size());
-
- }
-
-}
[08/30] apex-malhar git commit: Renamed demos to examples. Packages
and artifactid names are changed as suggested.
Posted by th...@apache.org.
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/Application.java
----------------------------------------------------------------------
diff --git a/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/Application.java b/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/Application.java
new file mode 100644
index 0000000..8132c7d
--- /dev/null
+++ b/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/Application.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.machinedata;
+
+import java.util.Map;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.apex.examples.machinedata.data.MachineKey;
+import org.apache.apex.examples.machinedata.operator.MachineInfoAveragingOperator;
+import org.apache.apex.examples.machinedata.operator.MachineInfoAveragingPrerequisitesOperator;
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.contrib.redis.RedisKeyValPairOutputOperator;
+import com.datatorrent.lib.io.SmtpOutputOperator;
+
+/**
+ * <p>
+ * Resource monitor application.
+ * </p>
+ *
+ * @since 0.3.5
+ */
+@ApplicationAnnotation(name = "MachineDataExample")
+@SuppressWarnings("unused")
+public class Application implements StreamingApplication
+{
+
+ private static final Logger LOG = LoggerFactory.getLogger(Application.class);
+
+ /**
+ * This function sets up the DAG for calculating the average
+ *
+ * @param dag the DAG instance
+ * @param conf the configuration instance
+ * @return MachineInfoAveragingPrerequisitesOperator
+ */
+ private MachineInfoAveragingPrerequisitesOperator addAverageCalculation(DAG dag, Configuration conf)
+ {
+ MachineInfoAveragingPrerequisitesOperator prereqAverageOper = dag.addOperator("Aggregator", MachineInfoAveragingPrerequisitesOperator.class);
+ MachineInfoAveragingOperator averageOperator = dag.addOperator("AverageCalculator", MachineInfoAveragingOperator.class);
+ RedisKeyValPairOutputOperator<MachineKey, Map<String, String>> redisAvgOperator = dag.addOperator("Persister", new RedisKeyValPairOutputOperator<MachineKey, Map<String, String>>());
+ dag.addStream("Average", averageOperator.outputPort, redisAvgOperator.input);
+ SmtpOutputOperator smtpOutputOperator = dag.addOperator("Alerter", new SmtpOutputOperator());
+ dag.addStream("Aggregates", prereqAverageOper.outputPort, averageOperator.inputPort);
+ dag.addStream("Alerts", averageOperator.smtpAlert, smtpOutputOperator.input);
+ return prereqAverageOper;
+ }
+
+ /**
+ * Create the DAG
+ */
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ InputReceiver randomGen = dag.addOperator("Receiver", InputReceiver.class);
+ DimensionGenerator dimensionGenerator = dag.addOperator("DimensionsGenerator", DimensionGenerator.class);
+ dag.addStream("Events", randomGen.outputInline, dimensionGenerator.inputPort);
+ MachineInfoAveragingPrerequisitesOperator prereqAverageOper = addAverageCalculation(dag, conf);
+ dag.addStream("DimensionalData", dimensionGenerator.outputInline, prereqAverageOper.inputPort);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/DimensionGenerator.java
----------------------------------------------------------------------
diff --git a/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/DimensionGenerator.java b/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/DimensionGenerator.java
new file mode 100644
index 0000000..6c717c2
--- /dev/null
+++ b/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/DimensionGenerator.java
@@ -0,0 +1,121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.machinedata;
+
+import org.apache.apex.examples.machinedata.data.MachineInfo;
+import org.apache.apex.examples.machinedata.data.MachineKey;
+import com.datatorrent.api.Context;
+import com.datatorrent.api.DefaultInputPort;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.common.util.BaseOperator;
+
+
+/**
+ * <p>
+ * Information tuple generator with randomness.
+ * </p>
+ *
+ * @since 0.3.5
+ */
+@SuppressWarnings("unused")
+public class DimensionGenerator extends BaseOperator
+{
+ public transient DefaultOutputPort<MachineInfo> outputInline = new DefaultOutputPort<>();
+ public transient DefaultOutputPort<MachineInfo> output = new DefaultOutputPort<>();
+ private int threshold = 90;
+
+ public final transient DefaultInputPort<MachineInfo> inputPort = new DefaultInputPort<MachineInfo>()
+ {
+
+ @Override
+ public void process(MachineInfo tuple)
+ {
+ emitDimensions(tuple);
+ }
+
+ };
+
+ @Override
+ public void setup(Context.OperatorContext context)
+ {
+ super.setup(context);
+ }
+
+ /**
+ * This returns the threshold value set
+ * @return
+ */
+ public int getThreshold()
+ {
+ return threshold;
+ }
+
+ /**
+ * This function sets the threshold value. This value is used to check the maximum value for cpu/ram/hdd
+ * @param threshold
+ */
+ public void setThreshold(int threshold)
+ {
+ this.threshold = threshold;
+ }
+
+ /**
+ * This function takes in the tuple from upstream operator and generates tuples with different dimension combinations
+ *
+ * @param tuple
+ */
+ private void emitDimensions(MachineInfo tuple)
+ {
+ MachineKey tupleKey = tuple.getMachineKey();
+
+ for (int i = 0; i < 64; i++) {
+ MachineKey machineKey = new MachineKey(tupleKey.getTimeKey(),tupleKey.getDay());
+ if ((i & 1) != 0) {
+ machineKey.setCustomer(tupleKey.getCustomer());
+ }
+ if ((i & 2) != 0) {
+ machineKey.setProduct(tupleKey.getProduct());
+ }
+ if ((i & 4) != 0) {
+ machineKey.setOs(tupleKey.getOs());
+ }
+ if ((i & 8) != 0) {
+ machineKey.setDeviceId(tupleKey.getDeviceId());
+ }
+ if ((i & 16) != 0) {
+ machineKey.setSoftware1(tupleKey.getSoftware1());
+ }
+ if ((i & 32) != 0) {
+ machineKey.setSoftware2(tupleKey.getSoftware2());
+ }
+
+ int cpu = tuple.getCpu();
+ int ram = tuple.getRam();
+ int hdd = tuple.getHdd();
+ MachineInfo machineInfo = new MachineInfo();
+ machineInfo.setMachineKey(machineKey);
+ machineInfo.setCpu((cpu < threshold) ? cpu : threshold);
+ machineInfo.setRam((ram < threshold) ? ram : threshold);
+ machineInfo.setHdd((hdd < threshold) ? hdd : threshold);
+ outputInline.emit(machineInfo);
+ output.emit(machineInfo);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/InputReceiver.java
----------------------------------------------------------------------
diff --git a/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/InputReceiver.java b/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/InputReceiver.java
new file mode 100644
index 0000000..263db55
--- /dev/null
+++ b/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/InputReceiver.java
@@ -0,0 +1,524 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.machinedata;
+
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.Random;
+import java.util.TimeZone;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.apex.examples.machinedata.data.MachineInfo;
+import org.apache.apex.examples.machinedata.data.MachineKey;
+
+import com.datatorrent.api.Context;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.InputOperator;
+import com.datatorrent.common.util.BaseOperator;
+
+/**
+ * <p>
+ * Information tuple generator with randomness.
+ * </p>
+ *
+ * @since 0.3.5
+ */
+@SuppressWarnings("unused")
+public class InputReceiver extends BaseOperator implements InputOperator
+{
+ private static final Logger logger = LoggerFactory.getLogger(InputReceiver.class);
+
+ public transient DefaultOutputPort<MachineInfo> outputInline = new DefaultOutputPort<>();
+ private final Random randomGen = new Random();
+
+ private int customerMin = 1;
+ private int customerMax = 5;
+ private int productMin = 4;
+ private int productMax = 6;
+ private int osMin = 10;
+ private int osMax = 12;
+ private int software1Min = 10;
+ private int software1Max = 12;
+ private int software2Min = 12;
+ private int software2Max = 14;
+ private int software3Min = 4;
+ private int software3Max = 6;
+
+ private int deviceIdMin = 1;
+ private int deviceIdMax = 50;
+
+ private int tupleBlastSize = 1001;
+
+ private static final DateFormat minuteDateFormat = new SimpleDateFormat("HHmm");
+ private static final DateFormat dayDateFormat = new SimpleDateFormat("d");
+
+ static {
+ TimeZone tz = TimeZone.getTimeZone("GMT");
+ minuteDateFormat.setTimeZone(tz);
+ dayDateFormat.setTimeZone(tz);
+
+ }
+
+ @Override
+ public void setup(Context.OperatorContext context)
+ {
+ super.setup(context);
+ }
+
+ @Override
+ public void beginWindow(long windowId)
+ {
+ super.beginWindow(windowId);
+ }
+
+ @Override
+ public void emitTuples()
+ {
+ int count = 0;
+ Calendar calendar = Calendar.getInstance();
+ Date date = calendar.getTime();
+ String timeKey = minuteDateFormat.format(date);
+ String day = dayDateFormat.format(date);
+
+ while (count < tupleBlastSize) {
+ randomGen.setSeed(System.currentTimeMillis());
+
+ int customerVal = genCustomerId();
+ int productVal = genProductVer();
+ int osVal = genOsVer();
+ int software1Val = genSoftware1Ver();
+ int software2Val = genSoftware2Ver();
+ int software3Val = genSoftware3Ver();
+ int deviceIdVal = genDeviceId();
+
+ int cpuVal = genCpu(calendar);
+ int ramVal = genRam(calendar);
+ int hddVal = genHdd(calendar);
+
+ MachineKey machineKey = new MachineKey(timeKey, day);
+
+ machineKey.setCustomer(customerVal);
+ machineKey.setProduct(productVal);
+ machineKey.setOs(osVal);
+ machineKey.setDeviceId(deviceIdVal);
+ machineKey.setSoftware1(software1Val);
+ machineKey.setSoftware2(software2Val);
+ machineKey.setSoftware3(software3Val);
+ MachineInfo machineInfo = new MachineInfo();
+ machineInfo.setMachineKey(machineKey);
+ machineInfo.setCpu(cpuVal);
+ machineInfo.setRam(ramVal);
+ machineInfo.setHdd(hddVal);
+
+ outputInline.emit(machineInfo);
+
+ count++;
+ }
+ }
+
+ private int genCustomerId()
+ {
+ int range = customerMax - customerMin + 1;
+ return customerMin + randomGen.nextInt(range);
+ }
+
+ private int genProductVer()
+ {
+ int range = productMax - productMin + 1;
+ return productMin + randomGen.nextInt(range);
+ }
+
+ private int genOsVer()
+ {
+ int range = osMax - osMin + 1;
+ return osMin + randomGen.nextInt(range);
+ }
+
+ private int genSoftware3Ver()
+ {
+ int range = software3Max - software3Min + 1;
+ return software3Min + randomGen.nextInt(range);
+ }
+
+ private int genDeviceId()
+ {
+ int range = deviceIdMax - deviceIdMin + 1;
+ return deviceIdMin + randomGen.nextInt(range);
+ }
+
+ private int genSoftware1Ver()
+ {
+ int range = software1Max - software1Min + 1;
+ return software1Min + randomGen.nextInt(range);
+ }
+
+ private int genSoftware2Ver()
+ {
+ int range = software2Max - software2Min + 1;
+ return software2Min + randomGen.nextInt(range);
+ }
+
+ private int genCpu(Calendar cal)
+ {
+ int minute = cal.get(Calendar.MINUTE);
+ int second;
+ int range = minute / 2 + 19;
+ if (minute / 17 == 0) {
+ second = cal.get(Calendar.SECOND);
+ return (30 + randomGen.nextInt(range) + (minute % 7) - (second % 11));
+ } else if (minute / 47 == 0) {
+ second = cal.get(Calendar.SECOND);
+ return (7 + randomGen.nextInt(range) + (minute % 7) - (second % 7));
+ } else {
+ second = cal.get(Calendar.SECOND);
+ return (randomGen.nextInt(range) + (minute % 19) + (second % 7));
+ }
+ }
+
+ private int genRam(Calendar cal)
+ {
+ int minute = cal.get(Calendar.MINUTE);
+ int second;
+ int range = minute + 1;
+ if (minute / 23 == 0) {
+ second = cal.get(Calendar.SECOND);
+ return (20 + randomGen.nextInt(range) + (minute % 5) - (second % 11));
+ } else if (minute / 37 == 0) {
+ second = cal.get(Calendar.SECOND);
+ return (11 + randomGen.nextInt(60) - (minute % 5) - (second % 11));
+ } else {
+ second = cal.get(Calendar.SECOND);
+ return (randomGen.nextInt(range) + (minute % 17) + (second % 11));
+ }
+ }
+
+ private int genHdd(Calendar cal)
+ {
+ int minute = cal.get(Calendar.MINUTE);
+ int second;
+ int range = minute / 2 + 1;
+ if (minute / 37 == 0) {
+ second = cal.get(Calendar.SECOND);
+ return (25 + randomGen.nextInt(range) - minute % 7 - second % 11);
+ } else {
+ second = cal.get(Calendar.SECOND);
+ return (randomGen.nextInt(range) + minute % 23 + second % 11);
+ }
+ }
+
+ /**
+ * This method returns the minimum value for customer
+ *
+ * @return
+ */
+ public int getCustomerMin()
+ {
+ return customerMin;
+ }
+
+ /**
+ * This method is used to set the minimum value for customer
+ *
+ * @param customerMin the minimum customer value
+ */
+ public void setCustomerMin(int customerMin)
+ {
+ this.customerMin = customerMin;
+ }
+
+ /**
+ * This method returns the max value for customer
+ *
+ * @return
+ */
+ public int getCustomerMax()
+ {
+ return customerMax;
+ }
+
+ /**
+ * This method is used to set the max value for customer
+ *
+ * @param customerMax the max customer value
+ */
+ public void setCustomerMax(int customerMax)
+ {
+ this.customerMax = customerMax;
+ }
+
+ /**
+ * This method returns the minimum value for product
+ *
+ * @return
+ */
+ public int getProductMin()
+ {
+ return productMin;
+ }
+
+ /**
+ * This method is used to set the minimum value for product
+ *
+ * @param productMin the minimum product value
+ */
+ public void setProductMin(int productMin)
+ {
+ this.productMin = productMin;
+ }
+
+ /**
+ * This method returns the max value for product
+ *
+ * @return
+ */
+ public int getProductMax()
+ {
+ return productMax;
+ }
+
+ /**
+ * This method is used to set the max value for product
+ *
+ * @param productMax the max product value
+ */
+ public void setProductMax(int productMax)
+ {
+ this.productMax = productMax;
+ }
+
+ /**
+ * This method returns the minimum value for OS
+ *
+ * @return
+ */
+ public int getOsMin()
+ {
+ return osMin;
+ }
+
+ /**
+ * This method is used to set the minimum value for OS
+ *
+ * @param osMin the min OS value
+ */
+ public void setOsMin(int osMin)
+ {
+ this.osMin = osMin;
+ }
+
+ /**
+ * This method returns the max value for OS
+ *
+ * @return
+ */
+ public int getOsMax()
+ {
+ return osMax;
+ }
+
+ /**
+ * This method is used to set the max value for OS
+ *
+ * @param osMax the max OS value
+ */
+ public void setOsMax(int osMax)
+ {
+ this.osMax = osMax;
+ }
+
+ /**
+ * This method returns the minimum value for software1
+ *
+ * @return
+ */
+ public int getSoftware1Min()
+ {
+ return software1Min;
+ }
+
+ /**
+ * This method is used to set the minimum value for software1
+ *
+ * @param software1Min the minimum software1 value
+ */
+ public void setSoftware1Min(int software1Min)
+ {
+ this.software1Min = software1Min;
+ }
+
+ /**
+ * This method returns the max value for software1
+ *
+ * @return
+ */
+ public int getSoftware1Max()
+ {
+ return software1Max;
+ }
+
+ /**
+ * This method is used to set the max value for software1
+ *
+ * @param software1Max the max software1 value
+ */
+ public void setSoftware1Max(int software1Max)
+ {
+ this.software1Max = software1Max;
+ }
+
+ /**
+ * This method returns the minimum value for software2
+ *
+ * @return
+ */
+ public int getSoftware2Min()
+ {
+ return software2Min;
+ }
+
+ /**
+ * This method is used to set the minimum value for software2
+ *
+ * @param software2Min the minimum software2 value
+ */
+ public void setSoftware2Min(int software2Min)
+ {
+ this.software2Min = software2Min;
+ }
+
+ /**
+ * This method returns the max value for software2
+ *
+ * @return
+ */
+ public int getSoftware2Max()
+ {
+ return software2Max;
+ }
+
+ /**
+ * This method is used to set the max value for software2
+ *
+ * @param software2Max the max software2 value
+ */
+ public void setSoftware2Max(int software2Max)
+ {
+ this.software2Max = software2Max;
+ }
+
+ /**
+ * This method returns the minimum value for software3
+ *
+ * @return
+ */
+ public int getSoftware3Min()
+ {
+ return software3Min;
+ }
+
+ /**
+ * This method is used to set the minimum value for software3
+ *
+ * @param software3Min the minimum software3 value
+ */
+ public void setSoftware3Min(int software3Min)
+ {
+ this.software3Min = software3Min;
+ }
+
+ /**
+ * This method returns the max value for software3
+ *
+ * @return
+ */
+ public int getSoftware3Max()
+ {
+ return software3Max;
+ }
+
+ /**
+ * This method is used to set the max value for software3
+ *
+ * @param software3Max the max software3 value
+ */
+ public void setSoftware3Max(int software3Max)
+ {
+ this.software3Max = software3Max;
+ }
+
+ /**
+ * This method returns the minimum value for deviceId
+ *
+ * @return
+ */
+ public int getDeviceIdMin()
+ {
+ return deviceIdMin;
+ }
+
+ /**
+ * This method is used to set the minimum value for deviceId
+ *
+ * @param deviceIdMin the minimum deviceId value
+ */
+ public void setDeviceIdMin(int deviceIdMin)
+ {
+ this.deviceIdMin = deviceIdMin;
+ }
+
+ /**
+ * This method returns the max value for deviceId
+ *
+ * @return
+ */
+ public int getDeviceIdMax()
+ {
+ return deviceIdMax;
+ }
+
+ /**
+ * This method is used to set the max value for deviceId
+ *
+ * @param deviceIdMax the max deviceId value
+ */
+ public void setDeviceIdMax(int deviceIdMax)
+ {
+ this.deviceIdMax = deviceIdMax;
+ }
+
+ /**
+ * @return the tupleBlastSize
+ */
+ public int getTupleBlastSize()
+ {
+ return tupleBlastSize;
+ }
+
+ /**
+ * @param tupleBlastSize the tupleBlastSize to set
+ */
+ public void setTupleBlastSize(int tupleBlastSize)
+ {
+ this.tupleBlastSize = tupleBlastSize;
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/data/AverageData.java
----------------------------------------------------------------------
diff --git a/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/data/AverageData.java b/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/data/AverageData.java
new file mode 100644
index 0000000..3c12335
--- /dev/null
+++ b/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/data/AverageData.java
@@ -0,0 +1,107 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.machinedata.data;
+
+
+/**
+ * This class stores the value of sum and the count of values summed.
+ * <p>
+ * AverageData class.
+ * </p>
+ *
+ * @since 0.3.5
+ */
+public class AverageData
+{
+
+ private long cpu;
+ private long hdd;
+ private long ram;
+ private long count;
+
+ /**
+ * This is default constructor that sets the sum and count to 0
+ */
+ public AverageData()
+ {
+
+ }
+
+ /**
+ * This constructor takes the value of sum and count and initialize the local attributes to corresponding values
+ *
+ * @param count
+ * the value of count
+ */
+ public AverageData(long cpu,long hdd,long ram, long count)
+ {
+ this.cpu = cpu;
+ this.ram = ram;
+ this.hdd = hdd;
+ this.count = count;
+ }
+
+ public long getCpu()
+ {
+ return cpu;
+ }
+
+ public void setCpu(long cpu)
+ {
+ this.cpu = cpu;
+ }
+
+ public long getHdd()
+ {
+ return hdd;
+ }
+
+ public void setHdd(long hdd)
+ {
+ this.hdd = hdd;
+ }
+
+ public long getRam()
+ {
+ return ram;
+ }
+
+ public void setRam(long ram)
+ {
+ this.ram = ram;
+ }
+
+ /**
+ * This returns the value of count
+ * @return
+ */
+ public long getCount()
+ {
+ return count;
+ }
+
+ /**
+ * This method sets the value of count
+ * @param count
+ */
+ public void setCount(long count)
+ {
+ this.count = count;
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/data/MachineInfo.java
----------------------------------------------------------------------
diff --git a/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/data/MachineInfo.java b/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/data/MachineInfo.java
new file mode 100644
index 0000000..3952b70
--- /dev/null
+++ b/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/data/MachineInfo.java
@@ -0,0 +1,158 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.machinedata.data;
+
+/**
+ * This class stores the cpu% usage, ram% usage, hdd% usage and key information about a particular machine
+ * <p>
+ * MachineInfo class.
+ * </p>
+ *
+ * @since 0.3.5
+ */
+public class MachineInfo
+{
+ private MachineKey machineKey;
+ private int cpu;
+ private int ram;
+ private int hdd;
+
+ /**
+ * This default constructor
+ */
+ public MachineInfo()
+ {
+ }
+
+ /**
+ * This constructor takes MachineKey as input and initialize local attributes
+ *
+ * @param machineKey
+ * the MachineKey instance
+ */
+ public MachineInfo(MachineKey machineKey)
+ {
+ this.machineKey = machineKey;
+ }
+
+ /**
+ * This constructor takes MachineKey, cpu usage, ram usage, hdd usage as input and initialize local attributes
+ *
+ * @param machineKey
+ * the MachineKey instance
+ * @param cpu
+ * the CPU% usage
+ * @param ram
+ * the RAM% usage
+ * @param hdd
+ * the HDD% usage
+ */
+ public MachineInfo(MachineKey machineKey, int cpu, int ram, int hdd)
+ {
+ this.machineKey = machineKey;
+ this.cpu = cpu;
+ this.ram = ram;
+ this.hdd = hdd;
+ }
+
+ /**
+ * This method returns the MachineKey
+ *
+ * @return
+ */
+ public MachineKey getMachineKey()
+ {
+ return machineKey;
+ }
+
+ /**
+ * This method sets the MachineKey
+ *
+ * @param machineKey
+ * the MachineKey instance
+ */
+ public void setMachineKey(MachineKey machineKey)
+ {
+ this.machineKey = machineKey;
+ }
+
+ /**
+ * This method returns the CPU% usage
+ *
+ * @return
+ */
+ public int getCpu()
+ {
+ return cpu;
+ }
+
+ /**
+ * This method sets the CPU% usage
+ *
+ * @param cpu
+ * the CPU% usage
+ */
+ public void setCpu(int cpu)
+ {
+ this.cpu = cpu;
+ }
+
+ /**
+ * This method returns the RAM% usage
+ *
+ * @return
+ */
+ public int getRam()
+ {
+ return ram;
+ }
+
+ /**
+ * This method sets the RAM% usage
+ *
+ * @param ram
+ * the RAM% usage
+ */
+ public void setRam(int ram)
+ {
+ this.ram = ram;
+ }
+
+ /**
+ * This method returns the HDD% usage
+ *
+ * @return
+ */
+ public int getHdd()
+ {
+ return hdd;
+ }
+
+ /**
+ * This method sets the HDD% usage
+ *
+ * @param hdd
+ * the HDD% usage
+ */
+ public void setHdd(int hdd)
+ {
+ this.hdd = hdd;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/data/MachineKey.java
----------------------------------------------------------------------
diff --git a/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/data/MachineKey.java b/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/data/MachineKey.java
new file mode 100644
index 0000000..2bf0a53
--- /dev/null
+++ b/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/data/MachineKey.java
@@ -0,0 +1,381 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.machinedata.data;
+
+/**
+ * This class stores the information about various softwares, deviceIds, OS of the device
+ * <p>
+ * MachineKey class.
+ * </p>
+ *
+ * @since 0.3.5
+ */
+public class MachineKey
+{
+
+ private Integer customer;
+ private Integer product;
+ private Integer os;
+ private Integer software1;
+ private Integer software2;
+ private Integer software3;
+ private Integer deviceId;
+ private String timeKey;
+ private String day;
+
+ /**
+ * This constructor takes the format in which time has to be captured and the day when this instance is created
+ *
+ * @param timeKey the format in which time has to be captured
+ * @param day the day when this instance is created
+ */
+ public MachineKey(String timeKey, String day)
+ {
+ this.timeKey = timeKey;
+ this.day = day;
+ }
+
+ /**
+ * This is default constructor
+ */
+ public MachineKey()
+ {
+ }
+
+ /**
+ * This constructor takes format in which time has to be captured, the day when this instance is created, the customer
+ * id, product Id on the device, OS version on the device, software1 version on the device, software2 version on the device,
+ * software3 version on the device, deviceId on the device,
+ *
+ * @param timeKey the format in which time has to be captured
+ * @param day the day when this instance is created
+ * @param customer the customer Id
+ * @param product product Id
+ * @param os OS version
+ * @param software1 software1 version
+ * @param software2 software2 version
+ * @param software3 software3 version
+ * @param deviceId deviceId
+ */
+ public MachineKey(String timeKey, String day, Integer customer, Integer product, Integer os, Integer software1, Integer software2, Integer software3, Integer deviceId)
+ {
+ this.timeKey = timeKey;
+ this.day = day;
+ this.customer = customer;
+ this.product = product;
+ this.os = os;
+ this.software1 = software1;
+ this.software2 = software2;
+ this.software3 = software3;
+ this.deviceId = deviceId;
+ }
+
+ /**
+ * This method returns the format in which the time is captured. The time is the time when this instance of MachineKey
+ * was generated. For e.g. HHmm to capture Hour and minute
+ *
+ * @return
+ */
+ public String getTimeKey()
+ {
+ return timeKey;
+ }
+
+ /**
+ * This method sets the format in which the time is captured. The time is the time when this instance of MachineKey
+ * was generated. For e.g. HHmm to capture Hour and minute
+ *
+ * @param timeKey
+ * the value of format
+ */
+ public void setTimeKey(String timeKey)
+ {
+ this.timeKey = timeKey;
+ }
+
+ /**
+ * This method returns the day of the month when this instance of MachineKey was generated
+ *
+ * @return
+ */
+ public String getDay()
+ {
+ return day;
+ }
+
+ /**
+ * This method sets the day of the month when this instance of MachineKey was generated
+ *
+ * @param day
+ * the day of the month
+ */
+ public void setDay(String day)
+ {
+ this.day = day;
+ }
+
+ /**
+ * This method returns the customer Id
+ *
+ * @return
+ */
+ public Integer getCustomer()
+ {
+ return customer;
+ }
+
+ /**
+ * This method sets the customer Id
+ *
+ * @param customer
+ * the customer Id
+ */
+ public void setCustomer(Integer customer)
+ {
+ this.customer = customer;
+ }
+
+ /**
+ * This method returns product on the device
+ *
+ * @return
+ */
+ public Integer getProduct()
+ {
+ return product;
+ }
+
+ /**
+ * This method sets the product on the device
+ *
+ * @param product
+ * the value of product
+ */
+ public void setProduct(Integer product)
+ {
+ this.product = product;
+ }
+
+ /**
+ * This method returns the OS version on the device
+ *
+ * @return
+ */
+ public Integer getOs()
+ {
+ return os;
+ }
+
+ /**
+ * This method sets the OS version on the device
+ *
+ * @param os
+ * OS version
+ */
+ public void setOs(Integer os)
+ {
+ this.os = os;
+ }
+
+ /**
+ * This method returns the version of the software1 on the device
+ *
+ * @return
+ */
+ public Integer getSoftware1()
+ {
+ return software1;
+ }
+
+ /**
+ * This method sets the version of the software1 on the device
+ *
+ * @param software1 the version of the software1
+ */
+ public void setSoftware1(Integer software1)
+ {
+ this.software1 = software1;
+ }
+
+ /**
+ * This method returns the version of the software2 on the device
+ *
+ * @return
+ */
+ public Integer getSoftware2()
+ {
+ return software2;
+ }
+
+ /**
+ * This method sets the version of the software2 on the device
+ *
+ * @param software2
+ * the version of the software2
+ */
+ public void setSoftware2(Integer software2)
+ {
+ this.software2 = software2;
+ }
+
+ /**
+ * This method returns the version of the software3 on the device
+ *
+ * @return
+ */
+ public Integer getSoftware3()
+ {
+ return software3;
+ }
+
+ /**
+ * This method sets the version of the software3 on the device
+ *
+ * @param software3
+ * the version of the software3
+ */
+ public void setSoftware3(Integer software3)
+ {
+ this.software3 = software3;
+ }
+
+ @Override
+ public int hashCode()
+ {
+ int key = 0;
+ if (customer != null) {
+ key |= (1 << 31);
+ key ^= customer;
+ }
+ if (product != null) {
+ key |= (1 << 30);
+ key ^= product;
+ }
+ if (os != null) {
+ key |= (1 << 29);
+ key ^= os;
+ }
+ if (software1 != null) {
+ key |= (1 << 28);
+ key ^= software1;
+ }
+ if (software2 != null) {
+ key |= (1 << 27);
+ key ^= software2;
+ }
+ if (software3 != null) {
+ key |= (1 << 26);
+ key ^= software3;
+ }
+ if (deviceId != null) {
+ key |= (1 << 25);
+ key ^= deviceId;
+ }
+ if (timeKey != null) {
+ key |= (1 << 24);
+ key ^= timeKey.hashCode();
+ }
+ if (day != null) {
+ key |= (1 << 23);
+ key ^= day.hashCode();
+ }
+
+ return key;
+ }
+
+ @Override
+ public boolean equals(Object obj)
+ {
+ if (!(obj instanceof MachineKey)) {
+ return false;
+ }
+ MachineKey mkey = (MachineKey)obj;
+ return checkStringEqual(this.timeKey, mkey.timeKey) && checkStringEqual(this.day, mkey.day) && checkIntEqual(this.customer, mkey.customer) && checkIntEqual(this.product, mkey.product) && checkIntEqual(this.os, mkey.os) && checkIntEqual(this.software1, mkey.software1) && checkIntEqual(this.software2, mkey.software2) && checkIntEqual(this.software3, mkey.software3) && checkIntEqual(this.deviceId, mkey.deviceId);
+ }
+
+ private boolean checkIntEqual(Integer a, Integer b)
+ {
+ if ((a == null) && (b == null)) {
+ return true;
+ }
+ if ((a != null) && a.equals(b)) {
+ return true;
+ }
+ return false;
+ }
+
+ private boolean checkStringEqual(String a, String b)
+ {
+ if ((a == null) && (b == null)) {
+ return true;
+ }
+ if ((a != null) && a.equals(b)) {
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public String toString()
+ {
+ StringBuilder sb = new StringBuilder(timeKey);
+ if (customer != null) {
+ sb.append("|0:").append(customer);
+ }
+ if (product != null) {
+ sb.append("|1:").append(product);
+ }
+ if (os != null) {
+ sb.append("|2:").append(os);
+ }
+ if (software1 != null) {
+ sb.append("|3:").append(software1);
+ }
+ if (software2 != null) {
+ sb.append("|4:").append(software2);
+ }
+ if (software3 != null) {
+ sb.append("|5:").append(software3);
+ }
+ if (deviceId != null) {
+ sb.append("|6:").append(deviceId);
+ }
+ return sb.toString();
+ }
+
+ /**
+ * This method returns the deviceId of the device
+ * @return The deviceId
+ */
+ public Integer getDeviceId()
+ {
+ return deviceId;
+ }
+
+ /**
+ * This method sets the deviceId of the device
+ *
+ * @param deviceId
+ */
+ public void setDeviceId(Integer deviceId)
+ {
+ this.deviceId = deviceId;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/data/ResourceType.java
----------------------------------------------------------------------
diff --git a/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/data/ResourceType.java b/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/data/ResourceType.java
new file mode 100644
index 0000000..2fd457a
--- /dev/null
+++ b/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/data/ResourceType.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.machinedata.data;
+
+import java.util.Map;
+
+import com.google.common.collect.Maps;
+
+/**
+ * This class captures the resources whose usage is collected for each device
+ * <p>ResourceType class.</p>
+ *
+ * @since 0.3.5
+ */
+public enum ResourceType
+{
+
+ CPU("cpu"), RAM("ram"), HDD("hdd");
+
+ private static Map<String, ResourceType> descToResource = Maps.newHashMap();
+
+ static {
+ for (ResourceType type : ResourceType.values()) {
+ descToResource.put(type.desc, type);
+ }
+ }
+
+ private String desc;
+
+ private ResourceType(String desc)
+ {
+ this.desc = desc;
+ }
+
+ @Override
+ public String toString()
+ {
+ return desc;
+ }
+
+ /**
+ * This method returns ResourceType for the given description
+ * @param desc the description
+ * @return
+ */
+ public static ResourceType getResourceTypeOf(String desc)
+ {
+ return descToResource.get(desc);
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/operator/CalculatorOperator.java
----------------------------------------------------------------------
diff --git a/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/operator/CalculatorOperator.java b/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/operator/CalculatorOperator.java
new file mode 100644
index 0000000..34a9514
--- /dev/null
+++ b/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/operator/CalculatorOperator.java
@@ -0,0 +1,279 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.machinedata.operator;
+
+import java.io.Serializable;
+import java.math.BigDecimal;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import javax.validation.constraints.Max;
+import javax.validation.constraints.Min;
+
+import org.apache.apex.examples.machinedata.data.MachineInfo;
+import org.apache.apex.examples.machinedata.data.MachineKey;
+import org.apache.apex.examples.machinedata.data.ResourceType;
+import org.apache.apex.examples.machinedata.util.DataTable;
+
+import com.google.common.base.Objects;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+import com.datatorrent.api.DefaultInputPort;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.StreamCodec;
+import com.datatorrent.common.util.BaseOperator;
+
+import com.datatorrent.lib.codec.KryoSerializableStreamCodec;
+import com.datatorrent.lib.util.KeyValPair;
+
+/**
+ * <p>
+ * CalculatorOperator class.
+ * </p>
+ *
+ * @since 0.3.5
+ */
+public class CalculatorOperator extends BaseOperator
+{
+
+ private final DataTable<MachineKey, ResourceType, List<Integer>> data = new DataTable<>();
+
+ @Min(1)
+ @Max(99)
+ private int kthPercentile = 95; // kth percentile
+ private boolean computePercentile;
+ private boolean computeSD;
+ private boolean computeMax;
+
+ private int percentileThreshold = 80;
+ private int sdThreshold = 70;
+ private int maxThreshold = 99;
+
+ public final transient DefaultInputPort<MachineInfo> dataPort = new DefaultInputPort<MachineInfo>()
+ {
+ @Override
+ public void process(MachineInfo tuple)
+ {
+ addDataToCache(tuple);
+ }
+
+ /**
+ * Stream codec used for partitioning.
+ */
+ @Override
+ public StreamCodec<MachineInfo> getStreamCodec()
+ {
+ return new MachineInfoStreamCodec();
+ }
+ };
+
+ public final transient DefaultOutputPort<KeyValPair<MachineKey, Map<ResourceType, Double>>> percentileOutputPort = new DefaultOutputPort<>();
+
+ public final transient DefaultOutputPort<KeyValPair<MachineKey, Map<ResourceType, Double>>> sdOutputPort = new DefaultOutputPort<>();
+
+ public final transient DefaultOutputPort<KeyValPair<MachineKey, Map<ResourceType, Integer>>> maxOutputPort = new DefaultOutputPort<>();
+
+ public transient DefaultOutputPort<String> smtpAlert = new DefaultOutputPort<>();
+
+ private void addDataToCache(MachineInfo tuple)
+ {
+ MachineKey machineKey = tuple.getMachineKey();
+ if (!data.containsRow(machineKey)) {
+ data.put(machineKey, ResourceType.CPU, Lists.<Integer>newArrayList());
+ data.put(machineKey, ResourceType.RAM, Lists.<Integer>newArrayList());
+ data.put(machineKey, ResourceType.HDD, Lists.<Integer>newArrayList());
+ }
+ data.get(machineKey, ResourceType.CPU).add(tuple.getCpu());
+ data.get(machineKey, ResourceType.RAM).add(tuple.getRam());
+ data.get(machineKey, ResourceType.HDD).add(tuple.getHdd());
+ }
+
+ @Override
+ public void endWindow()
+ {
+
+ if (computePercentile) {
+ for (MachineKey machineKey : data.rowKeySet()) {
+ Collections.sort(data.get(machineKey, ResourceType.CPU));
+ Collections.sort(data.get(machineKey, ResourceType.RAM));
+ Collections.sort(data.get(machineKey, ResourceType.HDD));
+
+ Map<ResourceType, Double> percentileData = Maps.newHashMap();
+ percentileData.put(ResourceType.CPU, getKthPercentile(data.get(machineKey, ResourceType.CPU)));
+ percentileData.put(ResourceType.RAM, getKthPercentile(data.get(machineKey, ResourceType.RAM)));
+ percentileData.put(ResourceType.HDD, getKthPercentile(data.get(machineKey, ResourceType.HDD)));
+ percentileOutputPort.emit(new KeyValPair<>(machineKey, percentileData));
+
+ for (ResourceType resourceType : percentileData.keySet()) {
+ double percentileValue = percentileData.get(resourceType);
+ if (percentileValue > percentileThreshold) {
+ emitAlert(resourceType, machineKey, percentileValue, "Percentile");
+ }
+ }
+ }
+ }
+ if (computeSD) {
+ for (MachineKey machineKey : data.rowKeySet()) {
+
+ Map<ResourceType, Double> sdData = Maps.newHashMap();
+
+ for (ResourceType resourceType : ResourceType.values()) {
+ sdData.put(resourceType, getSD(data.get(machineKey, resourceType)));
+ }
+ sdOutputPort.emit(new KeyValPair<>(machineKey, sdData));
+
+ for (ResourceType resourceType : sdData.keySet()) {
+ double sdValue = sdData.get(resourceType);
+ if (sdValue > sdThreshold) {
+ emitAlert(resourceType, machineKey, sdValue, "SD");
+ }
+ }
+ }
+ }
+ if (computeMax) {
+ for (MachineKey machineKey : data.rowKeySet()) {
+
+ Map<ResourceType, Integer> maxData = Maps.newHashMap();
+ maxData.put(ResourceType.CPU, Collections.max(data.get(machineKey, ResourceType.CPU)));
+ maxData.put(ResourceType.RAM, Collections.max(data.get(machineKey, ResourceType.RAM)));
+ maxData.put(ResourceType.HDD, Collections.max(data.get(machineKey, ResourceType.HDD)));
+
+ maxOutputPort.emit(new KeyValPair<>(machineKey, maxData));
+
+ for (ResourceType resourceType : maxData.keySet()) {
+ double sdValue = maxData.get(resourceType).doubleValue();
+ if (sdValue > maxThreshold) {
+ emitAlert(resourceType, machineKey, sdValue, "Max");
+ }
+ }
+ }
+ }
+ data.clear();
+ }
+
+ private void emitAlert(ResourceType type, MachineKey machineKey, double alertVal, String prefix)
+ {
+ BigDecimal decimalVal = new BigDecimal(alertVal);
+ decimalVal = decimalVal.setScale(2, BigDecimal.ROUND_HALF_UP);
+ String alertTime = machineKey.getDay() + machineKey.getTimeKey();
+ smtpAlert.emit(prefix + "-" + type.toString().toUpperCase() + " alert at " + alertTime + " " + type + " usage breached current usage: " + decimalVal.doubleValue() + "% threshold: " + percentileThreshold + "%\n\n" + machineKey);
+ }
+
+ private double getKthPercentile(List<Integer> sorted)
+ {
+
+ double val = (kthPercentile * sorted.size()) / 100.0;
+ if (val == (int)val) {
+ // Whole number
+ int idx = (int)val - 1;
+ return (sorted.get(idx) + sorted.get(idx + 1)) / 2.0;
+ } else {
+ int idx = (int)Math.round(val) - 1;
+ return sorted.get(idx);
+ }
+ }
+
+ private double getSD(List<Integer> data)
+ {
+ int sum = 0;
+ for (int i : data) {
+ sum += i;
+ }
+ double avg = sum / (data.size() * 1.0);
+ double sd = 0;
+ for (Integer point : data) {
+ sd += Math.pow(point - avg, 2);
+ }
+ return Math.sqrt(sd);
+ }
+
+ /**
+ * @param kVal the percentile which will be emitted by this operator
+ */
+ public void setKthPercentile(int kVal)
+ {
+ this.kthPercentile = kVal;
+ }
+
+ /**
+ * @param doCompute when true percentile will be computed
+ */
+ public void setComputePercentile(boolean doCompute)
+ {
+ this.computePercentile = doCompute;
+ }
+
+ /**
+ * @param doCompute when true standard deviation will be computed
+ */
+ public void setComputeSD(boolean doCompute)
+ {
+ this.computeSD = doCompute;
+ }
+
+ /**
+ * @param doCompute when true max will be computed
+ */
+ public void setComputeMax(boolean doCompute)
+ {
+ this.computeMax = doCompute;
+ }
+
+ /**
+ * @param threshold for percentile when breached will cause alert
+ */
+ public void setPercentileThreshold(int threshold)
+ {
+ this.percentileThreshold = threshold;
+ }
+
+ /**
+ * @param threshold for standard deviation when breached will cause alert
+ */
+ public void setSDThreshold(int threshold)
+ {
+ this.sdThreshold = threshold;
+ }
+
+ /**
+ * @param threshold for Max when breached will cause alert
+ */
+ public void setMaxThreshold(int threshold)
+ {
+ this.maxThreshold = threshold;
+ }
+
+ public static class MachineInfoStreamCodec extends KryoSerializableStreamCodec<MachineInfo> implements Serializable
+ {
+ public MachineInfoStreamCodec()
+ {
+ super();
+ }
+
+ @Override
+ public int getPartition(MachineInfo o)
+ {
+ return Objects.hashCode(o.getMachineKey().getCustomer(), o.getMachineKey().getOs(), o.getMachineKey().getProduct(), o.getMachineKey().getSoftware1(), o.getMachineKey().getSoftware2(), o.getMachineKey().getSoftware3());
+ }
+
+ private static final long serialVersionUID = 201411031403L;
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/operator/MachineInfoAveragingOperator.java
----------------------------------------------------------------------
diff --git a/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/operator/MachineInfoAveragingOperator.java b/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/operator/MachineInfoAveragingOperator.java
new file mode 100644
index 0000000..1de676f
--- /dev/null
+++ b/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/operator/MachineInfoAveragingOperator.java
@@ -0,0 +1,216 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.machinedata.operator;
+
+import java.math.BigDecimal;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.apex.examples.machinedata.data.AverageData;
+import org.apache.apex.examples.machinedata.data.MachineInfo;
+import org.apache.apex.examples.machinedata.data.MachineKey;
+
+import com.google.common.collect.Maps;
+
+import com.datatorrent.api.DefaultInputPort;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.common.util.BaseOperator;
+
+import com.datatorrent.lib.util.KeyHashValPair;
+import com.datatorrent.lib.util.KeyValPair;
+
+/**
+ * This class calculates the average for various resources across different devices for a given key
+ * <p>MachineInfoAveragingOperator class.</p>
+ *
+ * @since 0.9.0
+ */
+@SuppressWarnings("unused")
+public class MachineInfoAveragingOperator extends BaseOperator
+{
+
+ public static final String CPU = "cpu";
+ public static final String RAM = "ram";
+ public static final String HDD = "hdd";
+ public static final String DAY = "day";
+
+ private final transient Map<MachineKey, AverageData> dataMap = new HashMap<>();
+
+ public final transient DefaultOutputPort<KeyValPair<MachineKey, Map<String, String>>> outputPort = new DefaultOutputPort<>();
+
+ public transient DefaultOutputPort<String> smtpAlert = new DefaultOutputPort<>();
+
+ private int threshold = 95;
+
+ /**
+ * Buffer all the tuples as is till end window gets called
+ */
+ public final transient DefaultInputPort<KeyHashValPair<MachineKey, AverageData>> inputPort = new DefaultInputPort<KeyHashValPair<MachineKey, AverageData>>()
+ {
+
+ @Override
+ public void process(KeyHashValPair<MachineKey, AverageData> tuple)
+ {
+ addTuple(tuple);
+ }
+ };
+
+ /**
+ * This method returns the threshold value
+ *
+ * @return
+ */
+ public int getThreshold()
+ {
+ return threshold;
+ }
+
+ /**
+ * This method sets the threshold value. If the average usage for any Resource is above this for a given key, then the alert is sent
+ *
+ * @param threshold the threshold value
+ */
+ public void setThreshold(int threshold)
+ {
+ this.threshold = threshold;
+ }
+
+ /**
+ * This adds the given tuple to the dataMap
+ *
+ * @param tuple input tuple
+ */
+ private void addTuple(KeyHashValPair<MachineKey, AverageData> tuple)
+ {
+ MachineKey key = tuple.getKey();
+ dataMap.put(key, tuple.getValue());
+ }
+
+ @Override
+ public void endWindow()
+ {
+ for (Map.Entry<MachineKey, AverageData> entry : dataMap.entrySet()) {
+ MachineKey key = entry.getKey();
+ AverageData averageResultMap = entry.getValue();
+ Map<String, String> averageResult = Maps.newHashMap();
+ long count = averageResultMap.getCount();
+ double average = averageResultMap.getCpu() / count;
+ averageResult.put(CPU, average + "");
+ emitAlert(average, CPU, key);
+ average = averageResultMap.getHdd() / count;
+ averageResult.put(HDD, average + "");
+ emitAlert(average, HDD, key);
+ average = averageResultMap.getRam() / count;
+ averageResult.put(RAM, average + "");
+ emitAlert(average, RAM, key);
+ averageResult.put(DAY, key.getDay());
+ outputPort.emit(new KeyValPair<>(key, averageResult));
+ }
+ dataMap.clear();
+ }
+
+ private void emitAlert(double average, String resourceType, MachineKey key)
+ {
+ if (average > threshold) {
+ BigDecimal bd = new BigDecimal(average);
+ bd = bd.setScale(2, BigDecimal.ROUND_HALF_UP);
+ String stime = key.getDay() + key.getTimeKey();
+ String skey = getKeyInfo(key);
+ smtpAlert.emit(resourceType.toUpperCase() + " alert at " + stime + " " + resourceType + " usage breached current usage: " + bd.doubleValue() + "% threshold: " + threshold + "%\n\n" + skey);
+ }
+ }
+
+ /**
+ * This method is used to artificially generate alerts
+ *
+ * @param genAlert
+ */
+ public void setGenAlert(boolean genAlert)
+ {
+ Calendar calendar = Calendar.getInstance();
+ long timestamp = System.currentTimeMillis();
+ calendar.setTimeInMillis(timestamp);
+ DateFormat minuteDateFormat = new SimpleDateFormat("HHmm");
+ Date date = calendar.getTime();
+ String timeKey = minuteDateFormat.format(date);
+ DateFormat dayDateFormat = new SimpleDateFormat("dd");
+ String day = dayDateFormat.format(date);
+
+ MachineKey alertKey = new MachineKey(timeKey, day);
+ alertKey.setCustomer(1);
+ alertKey.setProduct(5);
+ alertKey.setOs(10);
+ alertKey.setSoftware1(12);
+ alertKey.setSoftware2(14);
+ alertKey.setSoftware3(6);
+
+ MachineInfo machineInfo = new MachineInfo();
+ machineInfo.setMachineKey(alertKey);
+ machineInfo.setCpu(threshold + 1);
+ machineInfo.setRam(threshold + 1);
+ machineInfo.setHdd(threshold + 1);
+
+ smtpAlert.emit("CPU Alert: CPU Usage threshold (" + threshold + ") breached: current % usage: " + getKeyInfo(alertKey));
+ smtpAlert.emit("RAM Alert: RAM Usage threshold (" + threshold + ") breached: current % usage: " + getKeyInfo(alertKey));
+ smtpAlert.emit("HDD Alert: HDD Usage threshold (" + threshold + ") breached: current % usage: " + getKeyInfo(alertKey));
+ }
+
+ /**
+ * This method returns the String for a given MachineKey instance
+ *
+ * @param key MachineKey instance that needs to be converted to string
+ * @return
+ */
+ private String getKeyInfo(MachineKey key)
+ {
+ StringBuilder sb = new StringBuilder();
+ if (key instanceof MachineKey) {
+ MachineKey mkey = (MachineKey)key;
+ Integer customer = mkey.getCustomer();
+ if (customer != null) {
+ sb.append("customer: " + customer + "\n");
+ }
+ Integer product = mkey.getProduct();
+ if (product != null) {
+ sb.append("product version: " + product + "\n");
+ }
+ Integer os = mkey.getOs();
+ if (os != null) {
+ sb.append("os version: " + os + "\n");
+ }
+ Integer software1 = mkey.getSoftware1();
+ if (software1 != null) {
+ sb.append("software1 version: " + software1 + "\n");
+ }
+ Integer software2 = mkey.getSoftware2();
+ if (software2 != null) {
+ sb.append("software2 version: " + software2 + "\n");
+ }
+ Integer software3 = mkey.getSoftware3();
+ if (software3 != null) {
+ sb.append("software3 version: " + software3 + "\n");
+ }
+ }
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/operator/MachineInfoAveragingPrerequisitesOperator.java
----------------------------------------------------------------------
diff --git a/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/operator/MachineInfoAveragingPrerequisitesOperator.java b/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/operator/MachineInfoAveragingPrerequisitesOperator.java
new file mode 100644
index 0000000..14c8d25
--- /dev/null
+++ b/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/operator/MachineInfoAveragingPrerequisitesOperator.java
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.machinedata.operator;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.apex.examples.machinedata.data.AverageData;
+import org.apache.apex.examples.machinedata.data.MachineInfo;
+import org.apache.apex.examples.machinedata.data.MachineKey;
+
+import com.datatorrent.api.DefaultInputPort;
+import com.datatorrent.api.DefaultOutputPort;
+
+import com.datatorrent.common.util.BaseOperator;
+import com.datatorrent.lib.util.KeyHashValPair;
+
+/**
+ * This class calculates the partial sum and count for tuples generated by upstream Operator
+ * <p> MachineInfoAveragingPrerequisitesOperator class. </p>
+ *
+ * @since 0.3.5
+ */
+public class MachineInfoAveragingPrerequisitesOperator extends BaseOperator
+{
+
+ // Aggregate sum of all values seen for a key.
+ private Map<MachineKey, AverageData> sums = new HashMap<>();
+
+ public final transient DefaultOutputPort<KeyHashValPair<MachineKey, AverageData>> outputPort = new DefaultOutputPort<KeyHashValPair<MachineKey, AverageData>>()
+ {
+ @Override
+ public Unifier<KeyHashValPair<MachineKey, AverageData>> getUnifier()
+ {
+ MachineInfoAveragingUnifier unifier = new MachineInfoAveragingUnifier();
+ return unifier;
+ }
+ };
+
+ public transient DefaultInputPort<MachineInfo> inputPort = new DefaultInputPort<MachineInfo>()
+ {
+
+ @Override
+ public void process(MachineInfo tuple)
+ {
+ MachineKey key = tuple.getMachineKey();
+ AverageData averageData = sums.get(key);
+ if (averageData == null) {
+ averageData = new AverageData(tuple.getCpu(), tuple.getHdd(), tuple.getRam(), 1);
+ sums.put(key, averageData);
+ } else {
+ averageData.setCpu(averageData.getCpu() + tuple.getCpu());
+ averageData.setRam(averageData.getRam() + tuple.getRam());
+ averageData.setHdd(averageData.getHdd() + tuple.getHdd());
+ averageData.setCount(averageData.getCount() + 1);
+ }
+ }
+ };
+
+ @Override
+ public void endWindow()
+ {
+
+ for (Map.Entry<MachineKey, AverageData> entry : sums.entrySet()) {
+ if (outputPort.isConnected()) {
+ outputPort.emit(new KeyHashValPair<>(entry.getKey(), entry.getValue()));
+ }
+ }
+ sums.clear();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/operator/MachineInfoAveragingUnifier.java
----------------------------------------------------------------------
diff --git a/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/operator/MachineInfoAveragingUnifier.java b/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/operator/MachineInfoAveragingUnifier.java
new file mode 100644
index 0000000..02aabe7
--- /dev/null
+++ b/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/operator/MachineInfoAveragingUnifier.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.machinedata.operator;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.apex.examples.machinedata.data.AverageData;
+import org.apache.apex.examples.machinedata.data.MachineKey;
+
+import com.datatorrent.api.Context.OperatorContext;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.Operator.Unifier;
+
+import com.datatorrent.lib.util.KeyHashValPair;
+
+/**
+ * This class calculates the partial sum and count for a given key
+ * <p>MachineInfoAveragingUnifier class.</p>
+ *
+ * @since 0.9.0
+ */
+public class MachineInfoAveragingUnifier implements Unifier<KeyHashValPair<MachineKey, AverageData>>
+{
+
+ private Map<MachineKey, AverageData> sums = new HashMap<>();
+ public final transient DefaultOutputPort<KeyHashValPair<MachineKey, AverageData>> outputPort = new DefaultOutputPort<>();
+
+ @Override
+ public void beginWindow(long arg0)
+ {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
+ public void endWindow()
+ {
+ for (Map.Entry<MachineKey, AverageData> entry : sums.entrySet()) {
+ outputPort.emit(new KeyHashValPair<>(entry.getKey(), entry.getValue()));
+ }
+ sums.clear();
+
+ }
+
+ @Override
+ public void setup(OperatorContext arg0)
+ {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
+ public void teardown()
+ {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
+ public void process(KeyHashValPair<MachineKey, AverageData> arg0)
+ {
+ MachineKey tupleKey = arg0.getKey();
+ AverageData averageData = sums.get(tupleKey);
+ AverageData tupleValue = arg0.getValue();
+ if (averageData == null) {
+ sums.put(tupleKey, tupleValue);
+ } else {
+ averageData.setCpu(averageData.getCpu() + tupleValue.getCpu());
+ averageData.setRam(averageData.getRam() + tupleValue.getRam());
+ averageData.setHdd(averageData.getHdd() + tupleValue.getHdd());
+ averageData.setCount(averageData.getCount() + tupleValue.getCount());
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/util/Combinatorics.java
----------------------------------------------------------------------
diff --git a/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/util/Combinatorics.java b/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/util/Combinatorics.java
new file mode 100644
index 0000000..0ff8985
--- /dev/null
+++ b/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/util/Combinatorics.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.machinedata.util;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Generate combinations of elements for the given array of elements.
+ *
+ * Implements nCr = n! / (r! * (n-r)!)
+ *
+ * @since 0.3.5
+ */
+public class Combinatorics<T>
+{
+
+ private T[] values;
+ private int size = -1;
+ private List<T> result;
+ private Map<Integer, List<T>> resultMap = new HashMap<Integer, List<T>>();
+ private int resultMapSize = 0;
+
+ /**
+ * Generates all possible combinations with all the sizes.
+ *
+ * @param values
+ */
+ public Combinatorics(T[] values)
+ {
+ this.values = values;
+ this.size = -1;
+ this.result = new ArrayList<>();
+ }
+
+ /**
+ * Generates all possible combinations with the given size.
+ *
+ * @param values
+ * @param size
+ */
+ public Combinatorics(T[] values, int size)
+ {
+ this.values = values;
+ this.size = size;
+ this.result = new ArrayList<>();
+ }
+
+ public Map<Integer, List<T>> generate()
+ {
+
+ if (size == -1) {
+ size = values.length;
+ for (int i = 1; i <= size; i++) {
+ int[] tmp = new int[i];
+ Arrays.fill(tmp, -1);
+ generateCombinations(0, 0, tmp);
+ }
+ } else {
+ int[] tmp = new int[size];
+ Arrays.fill(tmp, -1);
+ generateCombinations(0, 0, tmp);
+ }
+ return resultMap;
+ }
+
+ public void generateCombinations(int start, int depth, int[] tmp)
+ {
+ if (depth == tmp.length) {
+ for (int j = 0; j < depth; j++) {
+ result.add(values[tmp[j]]);
+ }
+ resultMap.put(++resultMapSize, result);
+ result = new ArrayList<>();
+ return;
+ }
+ for (int i = start; i < values.length; i++) {
+ tmp[depth] = i;
+ generateCombinations(i + 1, depth + 1, tmp);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/util/DataTable.java
----------------------------------------------------------------------
diff --git a/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/util/DataTable.java b/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/util/DataTable.java
new file mode 100644
index 0000000..a147268
--- /dev/null
+++ b/examples/machinedata/src/main/java/org/apache/apex/examples/machinedata/util/DataTable.java
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.machinedata.util;
+
+import java.util.Map;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+import com.google.common.collect.Maps;
+
+/**
+ * <p>DataTable class.</p>
+ *
+ * @since 0.3.5
+ */
+public class DataTable<R,C,E>
+{
+
+ //machineKey, [cpu,ram,hdd] -> value
+ private final Map<R,Map<C,E>> table = Maps.newHashMap();
+
+ public boolean containsRow(R rowKey)
+ {
+ return table.containsKey(rowKey);
+ }
+
+ public void put(R rowKey,C colKey, E entry)
+ {
+ if (!containsRow(rowKey)) {
+ table.put(rowKey, Maps.<C,E>newHashMap());
+ }
+ table.get(rowKey).put(colKey, entry);
+ }
+
+ @Nullable
+ public E get(R rowKey, C colKey)
+ {
+ if (!containsRow(rowKey)) {
+ return null;
+ }
+ return table.get(rowKey).get(colKey);
+ }
+
+ public Set<R> rowKeySet()
+ {
+ return table.keySet();
+ }
+
+ public void clear()
+ {
+ table.clear();
+ }
+
+ public Map<R,Map<C,E>> getTable()
+ {
+ return table;
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/machinedata/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/examples/machinedata/src/main/resources/META-INF/properties.xml b/examples/machinedata/src/main/resources/META-INF/properties.xml
new file mode 100644
index 0000000..4ceb6b9
--- /dev/null
+++ b/examples/machinedata/src/main/resources/META-INF/properties.xml
@@ -0,0 +1,139 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<configuration>
+ <property>
+ <name>dt.attr.STREAMING_WINDOW_SIZE_MILLIS</name>
+ <value>1000</value>
+ </property>
+ <property>
+ <name>dt.application.MachineDataExample.operator.Receiver.attr.APPLICATION_WINDOW_COUNT
+ </name>
+ <value>5</value>
+ </property>
+ <property>
+ <name>dt.application.MachineDataExample.operator.Receiver.attr.PARTITIONER</name>
+ <value>com.datatorrent.common.partitioner.StatelessPartitioner:1</value>
+ </property>
+ <property>
+ <name>dt.application.MachineDataExample.operator.DimensionsGenerator.attr.APPLICATION_WINDOW_COUNT
+ </name>
+ <value>5</value>
+ </property>
+ <property>
+ <name>dt.application.MachineDataExample.stream.Events.locality
+ </name>
+ <value>CONTAINER_LOCAL</value>
+ </property>
+ <property>
+ <name>dt.application.MachineDataExample.operator.DimensionsGenerator.inputport.inputPort.attr.PARTITION_PARALLEL
+ </name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>dt.application.MachineDataExample.operator.Aggregator.inputport.inputPort.attr.PARTITION_PARALLEL
+ </name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>dt.application.MachineDataExample.operator.Aggregator.attr.APPLICATION_WINDOW_COUNT
+ </name>
+ <value>5</value>
+ </property>
+ <property>
+ <name>dt.application.MachineDataExample.operator.AverageCalculator.attr.APPLICATION_WINDOW_COUNT
+ </name>
+ <value>5</value>
+ </property>
+ <property>
+ <name>dt.application.MachineDataExample.operator.Persister.inputport.input.attr.PARTITION_PARALLEL
+ </name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>dt.application.MachineDataExample.operator.Persister.attr.APPLICATION_WINDOW_COUNT
+ </name>
+ <value>5</value>
+ </property>
+ <property>
+ <name>dt.application.MachineDataExample.operator.Persister.store.dbIndex
+ </name>
+ <value>2</value>
+ </property>
+ <property>
+ <name>dt.application.MachineDataExample.operator.Persister.store.host
+ </name>
+ <value></value>
+ </property>
+ <property>
+ <name>dt.application.MachineDataExample.operator.Persister.store.port
+ </name>
+ <value>6379</value>
+ </property>
+ <property>
+ <name>dt.application.MachineDataExample.port.*.attr.QUEUE_CAPACITY</name>
+ <value>32000</value>
+ </property>
+ <property>
+ <name>dt.application.MachineDataExample.operator.Alerter.from
+ </name>
+ <value></value>
+ </property>
+ <property>
+ <name>dt.application.MachineDataExample.operator.Alerter.subject
+ </name>
+ <value>Alert!!!</value>
+ </property>
+ <property>
+ <name>dt.application.MachineDataExample.operator.Alerter.recipients.TO
+ </name>
+ <value></value>
+ </property>
+ <property>
+ <name>dt.application.MachineDataExample.operator.Alerter.content
+ </name>
+ <value>{}</value>
+ </property>
+ <property>
+ <name>dt.application.MachineDataExample.operator.Alerter.smtpHost
+ </name>
+ <value>localhost</value>
+ </property>
+ <property>
+ <name>dt.application.MachineDataExample.operator.Alerter.smtpPort
+ </name>
+ <value>25</value>
+ </property>
+ <property>
+ <name>dt.application.MachineDataExample.operator.Alerter.useSsl
+ </name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>dt.application.MachineDataExample.operator.Aggregator.outputport.outputPort.attr.UNIFIER_LIMIT
+ </name>
+ <value>8</value>
+ </property>
+ <property>
+ <name>dt.application.MachineDataExample.stream.DimensionalData.locality
+ </name>
+ <value>THREAD_LOCAL</value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/machinedata/src/test/java/org/apache/apex/examples/machinedata/CalculatorOperatorTest.java
----------------------------------------------------------------------
diff --git a/examples/machinedata/src/test/java/org/apache/apex/examples/machinedata/CalculatorOperatorTest.java b/examples/machinedata/src/test/java/org/apache/apex/examples/machinedata/CalculatorOperatorTest.java
new file mode 100644
index 0000000..c69a2aa
--- /dev/null
+++ b/examples/machinedata/src/test/java/org/apache/apex/examples/machinedata/CalculatorOperatorTest.java
@@ -0,0 +1,197 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.machinedata;
+
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.apex.examples.machinedata.data.MachineInfo;
+import org.apache.apex.examples.machinedata.data.MachineKey;
+import org.apache.apex.examples.machinedata.data.ResourceType;
+import org.apache.apex.examples.machinedata.operator.CalculatorOperator;
+
+
+import com.google.common.collect.ImmutableList;
+
+import com.datatorrent.lib.testbench.CollectorTestSink;
+import com.datatorrent.lib.util.KeyValPair;
+import com.datatorrent.lib.util.TimeBucketKey;
+
+/**
+ * @since 0.3.5
+ */
+public class CalculatorOperatorTest
+{
+ private static DateFormat minuteDateFormat = new SimpleDateFormat("HHmm");
+ private static Logger LOG = LoggerFactory.getLogger(CalculatorOperatorTest.class);
+
+ /**
+ * Test node logic emits correct results
+ */
+ @Test
+ public void testNodeProcessing() throws Exception
+ {
+ CalculatorOperator calculatorOperator = new CalculatorOperator();
+ calculatorOperator.setup(null);
+
+ calculatorOperator.setComputePercentile(true);
+ calculatorOperator.setComputeMax(true);
+ calculatorOperator.setComputeSD(true);
+
+ testPercentile(calculatorOperator);
+ }
+
+ public void testPercentile(CalculatorOperator oper)
+ {
+
+ CollectorTestSink sortSink = new CollectorTestSink();
+ oper.percentileOutputPort.setSink(sortSink);
+ oper.setKthPercentile(50);
+ Calendar calendar = Calendar.getInstance();
+ Date date = calendar.getTime();
+ String timeKey = minuteDateFormat.format(date);
+ String day = calendar.get(Calendar.DAY_OF_MONTH) + "";
+
+ Integer vs = new Integer(1);
+ MachineKey mk = new MachineKey(timeKey, day, vs, vs, vs, vs, vs, vs, vs);
+
+ oper.beginWindow(0);
+
+ MachineInfo info = new MachineInfo(mk, 1, 1, 1);
+ oper.dataPort.process(info);
+
+ info.setCpu(2);
+ oper.dataPort.process(info);
+
+ info.setCpu(3);
+ oper.dataPort.process(info);
+
+ oper.endWindow();
+
+ Assert.assertEquals("number emitted tuples", 1, sortSink.collectedTuples.size());
+ for (Object o : sortSink.collectedTuples) {
+ LOG.debug(o.toString());
+ KeyValPair<TimeBucketKey, Map<ResourceType, Double>> keyValPair = (KeyValPair<TimeBucketKey, Map<ResourceType, Double>>)o;
+ Assert.assertEquals("emitted value for 'cpu' was ", 2.0, keyValPair.getValue().get(ResourceType.CPU), 0);
+ Assert.assertEquals("emitted value for 'hdd' was ", 1.0, keyValPair.getValue().get(ResourceType.HDD), 0);
+ Assert.assertEquals("emitted value for 'ram' was ", 1.0, keyValPair.getValue().get(ResourceType.RAM), 0);
+
+ }
+ LOG.debug("Done percentile testing\n");
+
+ }
+
+ public void testStandarDeviation(CalculatorOperator oper)
+ {
+ CollectorTestSink sortSink = new CollectorTestSink();
+ oper.sdOutputPort.setSink(sortSink);
+ Calendar calendar = Calendar.getInstance();
+ Date date = calendar.getTime();
+ String timeKey = minuteDateFormat.format(date);
+ String day = calendar.get(Calendar.DAY_OF_MONTH) + "";
+
+ Integer vs = new Integer(1);
+ MachineKey mk = new MachineKey(timeKey, day, vs, vs, vs, vs, vs, vs, vs);
+
+ oper.beginWindow(0);
+
+ MachineInfo info = new MachineInfo(mk, 1, 1, 1);
+ oper.dataPort.process(info);
+
+ info.setCpu(2);
+ oper.dataPort.process(info);
+
+ info.setCpu(3);
+ oper.dataPort.process(info);
+
+ oper.endWindow();
+
+ Assert.assertEquals("number emitted tuples", 1, sortSink.collectedTuples.size());
+ for (Object o : sortSink.collectedTuples) {
+ LOG.debug(o.toString());
+ KeyValPair<TimeBucketKey, Map<ResourceType, Double>> keyValPair = (KeyValPair<TimeBucketKey, Map<ResourceType, Double>>)o;
+ Assert.assertEquals("emitted value for 'cpu' was ", getSD(ImmutableList.of(1, 2, 3)), keyValPair.getValue().get(ResourceType.CPU), 0);
+ Assert.assertEquals("emitted value for 'hdd' was ", getSD(ImmutableList.of(1, 1, 1)), keyValPair.getValue().get(ResourceType.HDD), 0);
+ Assert.assertEquals("emitted value for 'ram' was ", getSD(ImmutableList.of(1, 1, 1)), keyValPair.getValue().get(ResourceType.RAM), 0);
+
+ }
+ LOG.debug("Done sd testing\n");
+
+ }
+
+ private final double getSD(List<Integer> input)
+ {
+ int sum = 0;
+ for (int i : input) {
+ sum += i;
+ }
+ double avg = sum / (input.size() * 1.0);
+ double sd = 0;
+ for (Integer point : input) {
+ sd += Math.pow(point - avg, 2);
+ }
+ return Math.sqrt(sd);
+ }
+
+ public void testMax(CalculatorOperator oper)
+ {
+ CollectorTestSink sortSink = new CollectorTestSink();
+ oper.maxOutputPort.setSink(sortSink);
+ Calendar calendar = Calendar.getInstance();
+ Date date = calendar.getTime();
+ String timeKey = minuteDateFormat.format(date);
+ String day = calendar.get(Calendar.DAY_OF_MONTH) + "";
+
+ Integer vs = new Integer(1);
+ MachineKey mk = new MachineKey(timeKey, day, vs, vs, vs, vs, vs, vs, vs);
+
+ oper.beginWindow(0);
+
+ MachineInfo info = new MachineInfo(mk, 1, 1, 1);
+ oper.dataPort.process(info);
+
+ info.setCpu(2);
+ oper.dataPort.process(info);
+
+ info.setCpu(3);
+ oper.dataPort.process(info);
+
+ oper.endWindow();
+
+ Assert.assertEquals("number emitted tuples", 1, sortSink.collectedTuples.size());
+ for (Object o : sortSink.collectedTuples) {
+ LOG.debug(o.toString());
+ KeyValPair<TimeBucketKey, Map<ResourceType, Double>> keyValPair = (KeyValPair<TimeBucketKey, Map<ResourceType, Double>>)o;
+ Assert.assertEquals("emitted value for 'cpu' was ", 3, keyValPair.getValue().get(ResourceType.CPU), 0);
+ Assert.assertEquals("emitted value for 'hdd' was ", 1, keyValPair.getValue().get(ResourceType.HDD), 0);
+ Assert.assertEquals("emitted value for 'ram' was ", 1, keyValPair.getValue().get(ResourceType.RAM), 0);
+
+ }
+ LOG.debug("Done max testing\n");
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/machinedata/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/examples/machinedata/src/test/resources/log4j.properties b/examples/machinedata/src/test/resources/log4j.properties
new file mode 100644
index 0000000..cf0d19e
--- /dev/null
+++ b/examples/machinedata/src/test/resources/log4j.properties
@@ -0,0 +1,43 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+log4j.rootLogger=DEBUG,CONSOLE
+
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+log4j.appender.CONSOLE.threshold=${test.log.console.threshold}
+test.log.console.threshold=DEBUG
+
+log4j.appender.RFA=org.apache.log4j.RollingFileAppender
+log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
+log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+log4j.appender.RFA.File=/tmp/app.log
+
+# to enable, add SYSLOG to rootLogger
+log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender
+log4j.appender.SYSLOG.syslogHost=127.0.0.1
+log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout
+log4j.appender.SYSLOG.layout.conversionPattern=${dt.cid} %-5p [%t] %c{2} %x - %m%n
+log4j.appender.SYSLOG.Facility=LOCAL1
+
+log4j.logger.org=info
+#log4j.logger.org.apache.commons.beanutils=warn
+log4j.logger.com.datatorrent=debug
+log4j.logger.org.apache.apex=debug
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/mobile/pom.xml
----------------------------------------------------------------------
diff --git a/examples/mobile/pom.xml b/examples/mobile/pom.xml
new file mode 100644
index 0000000..cb40887
--- /dev/null
+++ b/examples/mobile/pom.xml
@@ -0,0 +1,64 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>malhar-examples-mobile</artifactId>
+ <packaging>jar</packaging>
+
+ <name>Apache Apex Malhar Mobile Example</name>
+ <description></description>
+
+ <parent>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-examples</artifactId>
+ <version>3.7.0-SNAPSHOT</version>
+ </parent>
+
+ <properties>
+ <skipTests>true</skipTests>
+ </properties>
+
+ <dependencies>
+ <!-- add your dependencies here -->
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-servlet</artifactId>
+ <version>8.1.10.v20130312</version>
+ <scope>test</scope>
+ <type>jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-websocket</artifactId>
+ <version>8.1.10.v20130312</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ <version>3.1</version>
+ <type>jar</type>
+ </dependency>
+ </dependencies>
+
+</project>
[18/30] apex-malhar git commit: Renamed demos to examples. Packages
and artifactid names are changed as suggested.
Posted by th...@apache.org.
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/sql/src/test/java/org/apache/apex/malhar/sql/sample/SQLApplicationWithModelFileTest.java
----------------------------------------------------------------------
diff --git a/demos/sql/src/test/java/org/apache/apex/malhar/sql/sample/SQLApplicationWithModelFileTest.java b/demos/sql/src/test/java/org/apache/apex/malhar/sql/sample/SQLApplicationWithModelFileTest.java
deleted file mode 100644
index 7bbb8ec..0000000
--- a/demos/sql/src/test/java/org/apache/apex/malhar/sql/sample/SQLApplicationWithModelFileTest.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.apex.malhar.sql.sample;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.TimeZone;
-
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import org.apache.hadoop.conf.Configuration;
-
-import com.google.common.base.Predicates;
-import com.google.common.collect.Collections2;
-import com.google.common.collect.Lists;
-
-import com.datatorrent.api.LocalMode;
-
-public class SQLApplicationWithModelFileTest
-{
- private TimeZone defaultTZ;
-
- @Before
- public void setUp() throws Exception
- {
- defaultTZ = TimeZone.getDefault();
- TimeZone.setDefault(TimeZone.getTimeZone("GMT"));
- }
-
- @After
- public void tearDown() throws Exception
- {
- TimeZone.setDefault(defaultTZ);
- }
-
- @Test
- public void test() throws Exception
- {
- LocalMode lma = LocalMode.newInstance();
- Configuration conf = new Configuration(false);
- conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
- conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties-SQLApplicationWithModelFile.xml"));
-
- SQLApplicationWithModelFile app = new SQLApplicationWithModelFile();
-
- lma.prepareDAG(app, conf);
-
- LocalMode.Controller lc = lma.getController();
-
- PrintStream originalSysout = System.out;
- final ByteArrayOutputStream baos = new ByteArrayOutputStream();
- System.setOut(new PrintStream(baos));
-
- lc.runAsync();
- waitTillStdoutIsPopulated(baos, 30000);
- lc.shutdown();
-
- System.setOut(originalSysout);
-
- String[] sout = baos.toString().split(System.lineSeparator());
- Collection<String> filter = Collections2.filter(Arrays.asList(sout), Predicates.containsPattern("Delta Record:"));
-
- String[] actualLines = filter.toArray(new String[filter.size()]);
- Assert.assertTrue(actualLines[0].contains("RowTime=Mon Feb 15 10:15:00 GMT 2016, Product=paint1"));
- Assert.assertTrue(actualLines[1].contains("RowTime=Mon Feb 15 10:16:00 GMT 2016, Product=paint2"));
- Assert.assertTrue(actualLines[2].contains("RowTime=Mon Feb 15 10:17:00 GMT 2016, Product=paint3"));
- Assert.assertTrue(actualLines[3].contains("RowTime=Mon Feb 15 10:18:00 GMT 2016, Product=paint4"));
- Assert.assertTrue(actualLines[4].contains("RowTime=Mon Feb 15 10:19:00 GMT 2016, Product=paint5"));
- Assert.assertTrue(actualLines[5].contains("RowTime=Mon Feb 15 10:10:00 GMT 2016, Product=abcde6"));
- }
-
- public static boolean waitTillStdoutIsPopulated(ByteArrayOutputStream baos, int timeout) throws InterruptedException,
- IOException
- {
- long now = System.currentTimeMillis();
- Collection<String> filter = Lists.newArrayList();
- while (System.currentTimeMillis() - now < timeout) {
- baos.flush();
- String[] sout = baos.toString().split(System.lineSeparator());
- filter = Collections2.filter(Arrays.asList(sout), Predicates.containsPattern("Delta Record:"));
- if (filter.size() != 0) {
- break;
- }
-
- Thread.sleep(500);
- }
-
- return (filter.size() != 0);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/sql/src/test/resources/input.csv
----------------------------------------------------------------------
diff --git a/demos/sql/src/test/resources/input.csv b/demos/sql/src/test/resources/input.csv
deleted file mode 100644
index c4786d1..0000000
--- a/demos/sql/src/test/resources/input.csv
+++ /dev/null
@@ -1,6 +0,0 @@
-15/02/2016 10:15:00 +0000,1,paint1,11
-15/02/2016 10:16:00 +0000,2,paint2,12
-15/02/2016 10:17:00 +0000,3,paint3,13
-15/02/2016 10:18:00 +0000,4,paint4,14
-15/02/2016 10:19:00 +0000,5,paint5,15
-15/02/2016 10:10:00 +0000,6,abcde6,16
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/sql/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/demos/sql/src/test/resources/log4j.properties b/demos/sql/src/test/resources/log4j.properties
deleted file mode 100644
index 8ea3cfe..0000000
--- a/demos/sql/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,50 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-log4j.rootLogger=DEBUG,CONSOLE
-
-log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
-log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
-log4j.appender.CONSOLE.threshold=WARN
-test.log.console.threshold=WARN
-
-log4j.appender.RFA=org.apache.log4j.RollingFileAppender
-log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
-log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
-log4j.appender.RFA.File=/tmp/app.log
-
-# to enable, add SYSLOG to rootLogger
-log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender
-log4j.appender.SYSLOG.syslogHost=127.0.0.1
-log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout
-log4j.appender.SYSLOG.layout.conversionPattern=${dt.cid} %-5p [%t] %c{2} %x - %m%n
-log4j.appender.SYSLOG.Facility=LOCAL1
-
-log4j.logger.org=info
-#log4j.logger.org.apache.commons.beanutils=warn
-log4j.logger.com.datatorrent=INFO
-log4j.logger.org.apache.apex=INFO
-
-log4j.logger.org.apache.calcite=WARN
-log4j.logger.org.apache.kafka=WARN
-log4j.logger.org.I0Itec.zkclient.ZkClient=WARN
-log4j.logger.org.apache.zookeeper=WARN
-log4j.logger.kafka=WARN
-log4j.logger.kafka.consumer=WARN
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/twitter/pom.xml
----------------------------------------------------------------------
diff --git a/demos/twitter/pom.xml b/demos/twitter/pom.xml
deleted file mode 100644
index 767d809..0000000
--- a/demos/twitter/pom.xml
+++ /dev/null
@@ -1,101 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>twitter-demo</artifactId>
- <packaging>jar</packaging>
-
- <name>Apache Apex Malhar Twitter Demo</name>
- <description>Twitter Rolling Top Words application demonstrates real-time computations over a sliding window.</description>
-
- <parent>
- <groupId>org.apache.apex</groupId>
- <artifactId>malhar-demos</artifactId>
- <version>3.7.0-SNAPSHOT</version>
- </parent>
-
- <properties>
- <skipTests>true</skipTests>
- </properties>
-
- <dependencies>
- <dependency>
- <!-- required by twitter demo -->
- <groupId>org.twitter4j</groupId>
- <artifactId>twitter4j-core</artifactId>
- <version>4.0.6</version>
- </dependency>
- <dependency>
- <!-- required by twitter demo -->
- <groupId>org.twitter4j</groupId>
- <artifactId>twitter4j-stream</artifactId>
- <version>4.0.6</version>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase</artifactId>
- <version>0.94.20</version>
- <type>jar</type>
- <exclusions>
- <exclusion>
- <groupId>*</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>mysql</groupId>
- <artifactId>mysql-connector-java</artifactId>
- <version>5.1.22</version>
- </dependency>
- <dependency>
- <groupId>org.apache.apex</groupId>
- <artifactId>malhar-contrib</artifactId>
- <version>${project.version}</version>
- <exclusions>
- <exclusion>
- <groupId>*</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk-kinesis</artifactId>
- <version>1.9.10</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- <version>2.4.4</version>
- <scope>runtime</scope>
- </dependency>
- <dependency>
- <groupId>it.unimi.dsi</groupId>
- <artifactId>fastutil</artifactId>
- <version>6.6.4</version>
- </dependency>
- </dependencies>
-
-</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/twitter/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/demos/twitter/src/assemble/appPackage.xml b/demos/twitter/src/assemble/appPackage.xml
deleted file mode 100644
index 4138cf2..0000000
--- a/demos/twitter/src/assemble/appPackage.xml
+++ /dev/null
@@ -1,59 +0,0 @@
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
- <id>appPackage</id>
- <formats>
- <format>jar</format>
- </formats>
- <includeBaseDirectory>false</includeBaseDirectory>
- <fileSets>
- <fileSet>
- <directory>${basedir}/target/</directory>
- <outputDirectory>/app</outputDirectory>
- <includes>
- <include>${project.artifactId}-${project.version}.jar</include>
- </includes>
- </fileSet>
- <fileSet>
- <directory>${basedir}/target/deps</directory>
- <outputDirectory>/lib</outputDirectory>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/site/conf</directory>
- <outputDirectory>/conf</outputDirectory>
- <includes>
- <include>*.xml</include>
- </includes>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/main/resources/META-INF</directory>
- <outputDirectory>/META-INF</outputDirectory>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/main/resources/app</directory>
- <outputDirectory>/app</outputDirectory>
- </fileSet>
- </fileSets>
-
-</assembly>
-
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/twitter/src/main/java/com/datatorrent/demos/twitter/KinesisHashtagsApplication.java
----------------------------------------------------------------------
diff --git a/demos/twitter/src/main/java/com/datatorrent/demos/twitter/KinesisHashtagsApplication.java b/demos/twitter/src/main/java/com/datatorrent/demos/twitter/KinesisHashtagsApplication.java
deleted file mode 100644
index b9d32ab..0000000
--- a/demos/twitter/src/main/java/com/datatorrent/demos/twitter/KinesisHashtagsApplication.java
+++ /dev/null
@@ -1,236 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.twitter;
-
-import java.net.URI;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.DAG.Locality;
-import com.datatorrent.api.Operator.InputPort;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-import com.datatorrent.contrib.kinesis.AbstractKinesisInputOperator;
-import com.datatorrent.contrib.kinesis.KinesisStringInputOperator;
-import com.datatorrent.contrib.kinesis.KinesisStringOutputOperator;
-import com.datatorrent.contrib.kinesis.ShardManager;
-import com.datatorrent.contrib.twitter.TwitterSampleInput;
-import com.datatorrent.lib.algo.UniqueCounter;
-import com.datatorrent.lib.io.ConsoleOutputOperator;
-import com.datatorrent.lib.io.PubSubWebSocketOutputOperator;
-
-/**
- * Twitter Demo Application: <br>
- * This demo application samples random public status from twitter, send to Hashtag
- * extractor and extract the status and send it into kinesis <br>
- * Get the records from kinesis and converts into Hashtags. Top 10 Hashtag(s) mentioned in
- * tweets in last 5 mins are displayed on every window count (500ms).<br>
- * <br>
- *
- * Real Time Calculation :<br>
- * This application calculates top 10 Hashtag mentioned in tweets in last 5
- * minutes across a 1% random tweet sampling on a rolling window basis.<br>
- * <br>
- * Before running this application, you need to have a <a href="https://dev.twitter.com/apps">Twitter API account</a>,
- * <a href="https://http://aws.amazon.com/">AWS Account</a> and configure the authentication details.
- * For launch from CLI, those go into ~/.dt/dt-site.xml:
- * <pre>
- * {@code
- * <?xml version="1.0" encoding="UTF-8"?>
- * <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
- * <configuration>
- *
- * <property> <name>dt.operator.TweetSampler.prop.consumerKey</name>
- * <value>TBD</value> </property>
- *
- * <property> <name>dt.operator.TweetSampler.prop.consumerSecret</name>
- * <value>TBD</value> </property>
- *
- * <property> <name>dt.operator.TweetSampler.prop.accessToken</name>
- * <value>TBD</value> </property>
- *
- * <property> <name>dt.operator.TweetSampler.prop.accessTokenSecret</name>
- * <value>TBD</value> </property>
- *
- * <property> <name>dt.operator.FromKinesis.streamName</name>
- * <value>TBD</value> </property>
- *
- * <property> <name>dt.operator.FromKinesis.accessKey</name>
- * <value>TBD</value> </property>
- *
- * <property> <name>dt.operator.FromKinesis.secretKey</name>
- * <value>TBD</value> </property>
- *
- * <property> <name>dt.operator.ToKinesis.streamName</name>
- * <value>TBD</value> </property>
- *
- * <property> <name>dt.operator.ToKinesis.accessKey</name>
- * <value>TBD</value> </property>
- *
- * <property> <name>dt.operator.ToKinesis.secretKey</name>
- * <value>TBD</value> </property>
- *
- * </configuration>
- * }
- * </pre>
- * Custom Attributes: <br>
- * <b>topCounts operator : <b>
- * <ul>
- * <li>Top Count : 10, number of top unique Hashtag to be reported.</li>
- * <li>Sliding window count : 600, report over last 5 min (600 * .5 / 60 mins)</li>
- * <li>window slide value : 1</li>
- * </ul>
- * <p>
- * Running Java Test or Main app in IDE:
- *
- * <pre>
- * LocalMode.runApp(new Application(), 600000); // 10 min run
- * </pre>
- *
- * Run Success : <br>
- * For successful deployment and run, user should see similar output on console as below:
- *
- * <pre>
- * topHashtags: {"string": "{\"gameinsight\":\"12\",\"android\":\"8\",\"WotaJKT48alay\":\"12\",\"TernyataJKT48CyberItuForoumSampah\":\"8\",\"NHL15Subban\":\"30\",\"JKT48CYBERKeracunanCUKRIK\":\"8\",\"verifydjzoodel\":\"59\",\"androidgames\":\"9\",\"\u0645\u0639_\u0627\u0644\u0644\u0647\":\"10\",\"Jct48cyberTAIBABI\":\"10\"}"}
- * topHashtags: {"string": "{\"gameinsight\":\"12\",\"android\":\"8\",\"WotaJKT48alay\":\"12\",\"TernyataJKT48CyberItuForoumSampah\":\"8\",\"NHL15Subban\":\"30\",\"JKT48CYBERKeracunanCUKRIK\":\"8\",\"verifydjzoodel\":\"59\",\"androidgames\":\"9\",\"\u0645\u0639_\u0627\u0644\u0644\u0647\":\"10\",\"Jct48cyberTAIBABI\":\"10\"}"}
- * topHashtags: {"string": "{\"gameinsight\":\"12\",\"android\":\"8\",\"WotaJKT48alay\":\"12\",\"TernyataJKT48CyberItuForoumSampah\":\"8\",\"NHL15Subban\":\"30\",\"JKT48CYBERKeracunanCUKRIK\":\"8\",\"verifydjzoodel\":\"59\",\"androidgames\":\"9\",\"\u0645\u0639_\u0627\u0644\u0644\u0647\":\"10\",\"Jct48cyberTAIBABI\":\"10\"}"}
- * topHashtags: {"string": "{\"gameinsight\":\"12\",\"android\":\"8\",\"WotaJKT48alay\":\"12\",\"TernyataJKT48CyberItuForoumSampah\":\"8\",\"NHL15Subban\":\"30\",\"JKT48CYBERKeracunanCUKRIK\":\"8\",\"verifydjzoodel\":\"59\",\"androidgames\":\"9\",\"\u0645\u0639_\u0627\u0644\u0644\u0647\":\"10\",\"Jct48cyberTAIBABI\":\"10\"}"}
- * topHashtags: {"string": "{\"gameinsight\":\"12\",\"android\":\"8\",\"WotaJKT48alay\":\"12\",\"TernyataJKT48CyberItuForoumSampah\":\"8\",\"NHL15Subban\":\"30\",\"JKT48CYBERKeracunanCUKRIK\":\"8\",\"verifydjzoodel\":\"59\",\"androidgames\":\"9\",\"\u0645\u0639_\u0627\u0644\u0644\u0647\":\"10\",\"Jct48cyberTAIBABI\":\"10\"}"}
- * topHashtags: {"string": "{\"gameinsight\":\"12\",\"android\":\"8\",\"WotaJKT48alay\":\"12\",\"TernyataJKT48CyberItuForoumSampah\":\"8\",\"NHL15Subban\":\"30\",\"JKT48CYBERKeracunanCUKRIK\":\"8\",\"verifydjzoodel\":\"59\",\"androidgames\":\"9\",\"\u0645\u0639_\u0627\u0644\u0644\u0647\":\"10\",\"Jct48cyberTAIBABI\":\"10\"}"}
- * topHashtags: {"string": "{\"gameinsight\":\"12\",\"android\":\"8\",\"WotaJKT48alay\":\"12\",\"TernyataJKT48CyberItuForoumSampah\":\"8\",\"NHL15Subban\":\"30\",\"JKT48CYBERKeracunanCUKRIK\":\"8\",\"verifydjzoodel\":\"59\",\"androidgames\":\"9\",\"\u0645\u0639_\u0627\u0644\u0644\u0647\":\"10\",\"Jct48cyberTAIBABI\":\"10\"}"}
- * topHashtags: {"string": "{\"gameinsight\":\"12\",\"android\":\"8\",\"WotaJKT48alay\":\"12\",\"TernyataJKT48CyberItuForoumSampah\":\"8\",\"NHL15Subban\":\"30\",\"JKT48CYBERKeracunanCUKRIK\":\"8\",\"verifydjzoodel\":\"59\",\"androidgames\":\"9\",\"\u0645\u0639_\u0627\u0644\u0644\u0647\":\"10\",\"Jct48cyberTAIBABI\":\"10\"}"}
- * topHashtags: {"string": "{\"gameinsight\":\"12\",\"android\":\"8\",\"WotaJKT48alay\":\"12\",\"TernyataJKT48CyberItuForoumSampah\":\"8\",\"NHL15Subban\":\"30\",\"JKT48CYBERKeracunanCUKRIK\":\"8\",\"verifydjzoodel\":\"59\",\"androidgames\":\"9\",\"\u0645\u0639_\u0627\u0644\u0644\u0647\":\"10\",\"Jct48cyberTAIBABI\":\"10\"}"}
- * topHashtags: {"string": "{\"gameinsight\":\"12\",\"android\":\"8\",\"WotaJKT48alay\":\"12\",\"TernyataJKT48CyberItuForoumSampah\":\"8\",\"NHL15Subban\":\"30\",\"JKT48CYBERKeracunanCUKRIK\":\"8\",\"verifydjzoodel\":\"59\",\"androidgames\":\"9\",\"\u0645\u0639_\u0627\u0644\u0644\u0647\":\"10\",\"Jct48cyberTAIBABI\":\"10\"}"}
- * 2013-06-17 14:38:55,201 [main] INFO stram.StramLocalCluster run - Application finished.
- * 2013-06-17 14:38:55,201 [container-2] INFO stram.StramChild processHeartbeatResponse - Received shutdown request
- * </pre>
- *
- * Scaling Options : <br>
- * User can scale application by setting intial partition size > 1 on count
- * unique operator. <br>
- * <br>
- *
- * Application DAG : <br>
- * <img src="doc-files/Application.gif" width=600px > <br>
- * <br>
- *
- * Streaming Window Size : 500ms(default) <br>
- * Operator Details : <br>
- * <ul>
- * <li><b>The twitterFeed operator : </b> This operator samples random public
- * statues from twitter and emits to application. <br>
- * Class : com.datatorrent.demos.twitter.TwitterSampleInput <br>
- * StateFull : No, window count 1 <br>
- * </li>
- * <li><b>The HashtagExtractor operator : </b> This operator extracts Hashtag from
- * random sampled statues from twitter. <br>
- * Class : {@link com.datatorrent.demos.twitter.TwitterStatusHashtagExtractor} <br>
- * StateFull : No, window count 1 <br>
- * </li>
- * <li><b>The outputOp operator : </b> This operator sent the tags into the kinesis. <br>
- * Class : {@link com.datatorrent.contrib.kinesis.KinesisStringOutputOperator} <br>
- * </li>
- * <li><b>The inputOp operator : </b> This operator fetches the records from kinesis and
- * converts into hastags and emits them. <br>
- * Class : {@link com.datatorrent.contrib.kinesis.KinesisStringOutputOperator} <br>
- * </li>
- * <li><b>The uniqueCounter operator : </b> This operator aggregates count for each
- * Hashtag extracted from random samples. <br>
- * Class : {@link com.datatorrent.lib.algo.UniqueCounter} <br>
- * StateFull : No, window count 1 <br>
- * </li>
- * <li><b> The topCounts operator : </b> This operator caluculates top Hashtag in last 1
- * min sliding window count 1. <br>
- * Class : com.datatorrent.lib.algo.WindowedTopCounter <br>
- * StateFull : Yes, sliding window count 120 (1 min) <br>
- * </li>
- * <li><b>The operator Console: </b> This operator just outputs the input tuples
- * to the console (or stdout). <br>
- * </li>
- * </ul>
- *
- * @since 2.0.0
- */
-@ApplicationAnnotation(name = "TwitterKinesisDemo")
-public class KinesisHashtagsApplication implements StreamingApplication
-{
- private final Locality locality = null;
-
- private InputPort<Object> consoleOutput(DAG dag, String operatorName)
- {
- String gatewayAddress = dag.getValue(DAG.GATEWAY_CONNECT_ADDRESS);
- if (!StringUtils.isEmpty(gatewayAddress)) {
- URI uri = URI.create("ws://" + gatewayAddress + "/pubsub");
- String topic = "demos.twitter." + operatorName;
- //LOG.info("WebSocket with gateway at: {}", gatewayAddress);
- PubSubWebSocketOutputOperator<Object> wsOut = dag.addOperator(operatorName, new PubSubWebSocketOutputOperator<Object>());
- wsOut.setUri(uri);
- wsOut.setTopic(topic);
- return wsOut.input;
- }
- ConsoleOutputOperator operator = dag.addOperator(operatorName, new ConsoleOutputOperator());
- operator.setStringFormat(operatorName + ": %s");
- return operator.input;
- }
-
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- // Setup the operator to get the data from twitter sample stream injected into the system.
- TwitterSampleInput twitterFeed = new TwitterSampleInput();
- twitterFeed = dag.addOperator("TweetSampler", twitterFeed);
-
- // Setup the operator to get the Hashtags extracted from the twitter statuses
- TwitterStatusHashtagExtractor HashtagExtractor = dag.addOperator("HashtagExtractor", TwitterStatusHashtagExtractor.class);
-
- //Setup the operator send the twitter statuses to kinesis
- KinesisStringOutputOperator outputOp = dag.addOperator("ToKinesis", new KinesisStringOutputOperator());
- outputOp.setBatchSize(500);
-
- // Feed the statuses from feed into the input of the Hashtag extractor.
- dag.addStream("TweetStream", twitterFeed.status, HashtagExtractor.input).setLocality(Locality.CONTAINER_LOCAL);
- // Start counting the Hashtags coming out of Hashtag extractor
- dag.addStream("SendToKinesis", HashtagExtractor.hashtags, outputOp.inputPort).setLocality(locality);
-
- //------------------------------------------------------------------------------------------
-
- KinesisStringInputOperator inputOp = dag.addOperator("FromKinesis", new KinesisStringInputOperator());
- ShardManager shardStats = new ShardManager();
- inputOp.setShardManager(shardStats);
- inputOp.getConsumer().setRecordsLimit(600);
- inputOp.setStrategy(AbstractKinesisInputOperator.PartitionStrategy.MANY_TO_ONE.toString());
-
- // Setup a node to count the unique Hashtags within a window.
- UniqueCounter<String> uniqueCounter = dag.addOperator("UniqueHashtagCounter", new UniqueCounter<String>());
-
- // Get the aggregated Hashtag counts and count them over last 5 mins.
- WindowedTopCounter<String> topCounts = dag.addOperator("TopCounter", new WindowedTopCounter<String>());
- topCounts.setTopCount(10);
- topCounts.setSlidingWindowWidth(600);
- topCounts.setDagWindowWidth(1);
-
- dag.addStream("TwittedHashtags", inputOp.outputPort, uniqueCounter.data).setLocality(locality);
-
- // Count unique Hashtags
- dag.addStream("UniqueHashtagCounts", uniqueCounter.count, topCounts.input).setLocality(locality);
- // Count top 10
- dag.addStream("TopHashtags", topCounts.output, consoleOutput(dag, "topHashtags")).setLocality(locality);
- }
-}
-
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/twitter/src/main/java/com/datatorrent/demos/twitter/SlidingContainer.java
----------------------------------------------------------------------
diff --git a/demos/twitter/src/main/java/com/datatorrent/demos/twitter/SlidingContainer.java b/demos/twitter/src/main/java/com/datatorrent/demos/twitter/SlidingContainer.java
deleted file mode 100644
index 8b9f447..0000000
--- a/demos/twitter/src/main/java/com/datatorrent/demos/twitter/SlidingContainer.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.twitter;
-
-import java.io.Serializable;
-
-/**
- * Developed for a demo<br>
- *
- * @param <T> Type of object for which sliding window is being maintained.
- * @since 0.3.2
- */
-public class SlidingContainer<T> implements Serializable
-{
- private static final long serialVersionUID = 201305291751L;
- T identifier;
- int totalCount;
- int position;
- int[] windowedCount;
-
- @SuppressWarnings("unused")
- private SlidingContainer()
- {
- /* needed for Kryo serialization */
- }
-
- public SlidingContainer(T identifier, int windowCount)
- {
- this.identifier = identifier;
- this.totalCount = 0;
- this.position = 0;
- windowedCount = new int[windowCount];
- }
-
- public void adjustCount(int i)
- {
- windowedCount[position] += i;
- }
-
- public void slide()
- {
- int currentCount = windowedCount[position];
- position = position == windowedCount.length - 1 ? 0 : position + 1;
- totalCount += currentCount - windowedCount[position];
- windowedCount[position] = 0;
- }
-
- @Override
- public String toString()
- {
- return identifier + " => " + totalCount;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/twitter/src/main/java/com/datatorrent/demos/twitter/TwitterDumpApplication.java
----------------------------------------------------------------------
diff --git a/demos/twitter/src/main/java/com/datatorrent/demos/twitter/TwitterDumpApplication.java b/demos/twitter/src/main/java/com/datatorrent/demos/twitter/TwitterDumpApplication.java
deleted file mode 100644
index 9edce64..0000000
--- a/demos/twitter/src/main/java/com/datatorrent/demos/twitter/TwitterDumpApplication.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.twitter;
-
-import java.sql.PreparedStatement;
-import java.sql.SQLException;
-
-import javax.annotation.Nonnull;
-
-import org.apache.hadoop.conf.Configuration;
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.DAG.Locality;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-
-import com.datatorrent.contrib.twitter.TwitterSampleInput;
-import com.datatorrent.lib.db.jdbc.AbstractJdbcTransactionableOutputOperator;
-
-import twitter4j.Status;
-
-/**
- * An application which connects to Twitter Sample Input and stores all the
- * tweets with their usernames in a mysql database. Please review the docs
- * for TwitterTopCounterApplication to setup your twitter credentials. You
- * may also be able to change JDBCStore credentials using config file.
- *
- * You will also need to create appropriate database and tables with the
- * following schema, also included in mysql.sql in resources:
- * <pre>
- * DROP TABLE if exists tweets;
- * CREATE TABLE tweets (
- * window_id LONG NOT NULL,
- * creation_date DATE,
- * text VARCHAR(256) NOT NULL,
- * userid VARCHAR(40) NOT NULL,
- * KEY ( userid, creation_date)
- * );
- *
- * drop table if exists dt_window_id_tracker;
- * CREATE TABLE dt_window_id_tracker (
- * dt_application_id VARCHAR(100) NOT NULL,
- * dt_operator_id int(11) NOT NULL,
- * dt_window_id bigint NOT NULL,
- * UNIQUE (dt_application_id, dt_operator_id, dt_window_id)
- * ) ENGINE=MyISAM DEFAULT CHARSET=latin1;
- * </pre>
- *
- * @since 0.9.4
- */
-@ApplicationAnnotation(name = "TwitterDumpDemo")
-public class TwitterDumpApplication implements StreamingApplication
-{
- public static class Status2Database extends AbstractJdbcTransactionableOutputOperator<Status>
- {
- public static final String INSERT_STATUS_STATEMENT = "insert into tweets (window_id, creation_date, text, userid) values (?, ?, ?, ?)";
-
- public Status2Database()
- {
- store.setMetaTable("dt_window_id_tracker");
- store.setMetaTableAppIdColumn("dt_application_id");
- store.setMetaTableOperatorIdColumn("dt_operator_id");
- store.setMetaTableWindowColumn("dt_window_id");
- }
-
- @Nonnull
- @Override
- protected String getUpdateCommand()
- {
- return INSERT_STATUS_STATEMENT;
- }
-
- @Override
- protected void setStatementParameters(PreparedStatement statement, Status tuple) throws SQLException
- {
- statement.setLong(1, currentWindowId);
-
- statement.setDate(2, new java.sql.Date(tuple.getCreatedAt().getTime()));
- statement.setString(3, tuple.getText());
- statement.setString(4, tuple.getUser().getScreenName());
- statement.addBatch();
- }
- }
-
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- //dag.setAttribute(DAGContext.APPLICATION_NAME, "TweetsDump");
-
- TwitterSampleInput twitterStream = dag.addOperator("TweetSampler", new TwitterSampleInput());
-
- //ConsoleOutputOperator dbWriter = dag.addOperator("DatabaseWriter", new ConsoleOutputOperator());
-
- Status2Database dbWriter = dag.addOperator("DatabaseWriter", new Status2Database());
- dbWriter.getStore().setDatabaseDriver("com.mysql.jdbc.Driver");
- dbWriter.getStore().setDatabaseUrl("jdbc:mysql://node6.morado.com:3306/twitter");
- dbWriter.getStore().setConnectionProperties("user:twitter");
-
- dag.addStream("Statuses", twitterStream.status, dbWriter.input).setLocality(Locality.CONTAINER_LOCAL);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/twitter/src/main/java/com/datatorrent/demos/twitter/TwitterDumpHBaseApplication.java
----------------------------------------------------------------------
diff --git a/demos/twitter/src/main/java/com/datatorrent/demos/twitter/TwitterDumpHBaseApplication.java b/demos/twitter/src/main/java/com/datatorrent/demos/twitter/TwitterDumpHBaseApplication.java
deleted file mode 100644
index 3adbbe0..0000000
--- a/demos/twitter/src/main/java/com/datatorrent/demos/twitter/TwitterDumpHBaseApplication.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.twitter;
-
-import java.nio.ByteBuffer;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.Put;
-
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.DAG.Locality;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-
-import com.datatorrent.contrib.hbase.AbstractHBasePutOutputOperator;
-import com.datatorrent.contrib.twitter.TwitterSampleInput;
-
-import twitter4j.Status;
-
-/**
- * An application which connects to Twitter Sample Input and stores all the
- * tweets with their usernames in a hbase database. Please review the docs
- * for TwitterTopCounterApplication to setup your twitter credentials.
- *
- * You need to create the HBase table to run this demo. Table name can be
- * configured but columnfamily must be 'cf' to make this demo simple and complied
- * with the mysql based demo.
- * create 'tablename', 'cf'
- *
- * </pre>
- *
- * @since 1.0.2
- */
-@ApplicationAnnotation(name = "TwitterDumpHBaseDemo")
-public class TwitterDumpHBaseApplication implements StreamingApplication
-{
-
- public static class Status2Hbase extends AbstractHBasePutOutputOperator<Status>
- {
-
- @Override
- public Put operationPut(Status t)
- {
- Put put = new Put(ByteBuffer.allocate(8).putLong(t.getCreatedAt().getTime()).array());
- put.add("cf".getBytes(), "text".getBytes(), t.getText().getBytes());
- put.add("cf".getBytes(), "userid".getBytes(), t.getText().getBytes());
- return put;
- }
-
- }
-
-
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- //dag.setAttribute(DAGContext.APPLICATION_NAME, "TweetsDump");
-
- TwitterSampleInput twitterStream = dag.addOperator("TweetSampler", new TwitterSampleInput());
-
- Status2Hbase hBaseWriter = dag.addOperator("DatabaseWriter", new Status2Hbase());
-
- dag.addStream("Statuses", twitterStream.status, hBaseWriter.input).setLocality(Locality.CONTAINER_LOCAL);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/twitter/src/main/java/com/datatorrent/demos/twitter/TwitterStatusHashtagExtractor.java
----------------------------------------------------------------------
diff --git a/demos/twitter/src/main/java/com/datatorrent/demos/twitter/TwitterStatusHashtagExtractor.java b/demos/twitter/src/main/java/com/datatorrent/demos/twitter/TwitterStatusHashtagExtractor.java
deleted file mode 100644
index d22db40..0000000
--- a/demos/twitter/src/main/java/com/datatorrent/demos/twitter/TwitterStatusHashtagExtractor.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.twitter;
-
-import com.datatorrent.api.DefaultInputPort;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.common.util.BaseOperator;
-
-import twitter4j.HashtagEntity;
-import twitter4j.Status;
-
-/**
- * <p>TwitterStatusHashtagExtractor class.</p>
- *
- * @since 1.0.2
- */
-public class TwitterStatusHashtagExtractor extends BaseOperator
-{
- public final transient DefaultOutputPort<String> hashtags = new DefaultOutputPort<String>();
- public final transient DefaultInputPort<Status> input = new DefaultInputPort<Status>()
- {
- @Override
- public void process(Status status)
- {
- HashtagEntity[] entities = status.getHashtagEntities();
- if (entities != null) {
- for (HashtagEntity he : entities) {
- if (he != null) {
- hashtags.emit(he.getText());
- }
- }
- }
- }
-
- };
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/twitter/src/main/java/com/datatorrent/demos/twitter/TwitterStatusURLExtractor.java
----------------------------------------------------------------------
diff --git a/demos/twitter/src/main/java/com/datatorrent/demos/twitter/TwitterStatusURLExtractor.java b/demos/twitter/src/main/java/com/datatorrent/demos/twitter/TwitterStatusURLExtractor.java
deleted file mode 100644
index 6dbc436..0000000
--- a/demos/twitter/src/main/java/com/datatorrent/demos/twitter/TwitterStatusURLExtractor.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.twitter;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.datatorrent.api.DefaultInputPort;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.common.util.BaseOperator;
-
-import twitter4j.Status;
-import twitter4j.URLEntity;
-
-/**
- * <p>TwitterStatusURLExtractor class.</p>
- *
- * @since 0.3.2
- */
-public class TwitterStatusURLExtractor extends BaseOperator
-{
- public final transient DefaultOutputPort<String> url = new DefaultOutputPort<String>();
- public final transient DefaultInputPort<Status> input = new DefaultInputPort<Status>()
- {
- @Override
- public void process(Status status)
- {
- URLEntity[] entities = status.getURLEntities();
- if (entities != null) {
- for (URLEntity ue: entities) {
- if (ue != null) { // see why we intermittently get NPEs
- url.emit((ue.getExpandedURL() == null ? ue.getURL() : ue.getExpandedURL()).toString());
- }
- }
- }
- }
- };
-
- private static final Logger LOG = LoggerFactory.getLogger(TwitterStatusURLExtractor.class);
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/twitter/src/main/java/com/datatorrent/demos/twitter/TwitterStatusWordExtractor.java
----------------------------------------------------------------------
diff --git a/demos/twitter/src/main/java/com/datatorrent/demos/twitter/TwitterStatusWordExtractor.java b/demos/twitter/src/main/java/com/datatorrent/demos/twitter/TwitterStatusWordExtractor.java
deleted file mode 100644
index e05a37a..0000000
--- a/demos/twitter/src/main/java/com/datatorrent/demos/twitter/TwitterStatusWordExtractor.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.twitter;
-
-import java.util.Arrays;
-import java.util.HashSet;
-
-import com.datatorrent.api.Context.OperatorContext;
-import com.datatorrent.api.DefaultInputPort;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.common.util.BaseOperator;
-
-/**
- * <p>TwitterStatusWordExtractor class.</p>
- *
- * @since 0.3.2
- */
-public class TwitterStatusWordExtractor extends BaseOperator
-{
- public HashSet<String> filterList;
-
- public final transient DefaultOutputPort<String> output = new DefaultOutputPort<String>();
- public final transient DefaultInputPort<String> input = new DefaultInputPort<String>()
- {
- @Override
- public void process(String text)
- {
- String[] strs = text.split(" ");
- if (strs != null) {
- for (String str : strs) {
- if (str != null && !filterList.contains(str) ) {
- output.emit(str);
- }
- }
- }
- }
- };
-
- @Override
- public void setup(OperatorContext context)
- {
- this.filterList = new HashSet<String>(Arrays.asList(new String[]{"", " ","I","you","the","a","to","as","he","him","his","her","she","me","can","for","of","and","or","but",
- "this","that","!",",",".",":","#","/","@","be","in","out","was","were","is","am","are","so","no","...","my","de","RT","on","que","la","i","your","it","have","with","?","when",
- "up","just","do","at","&","-","+","*","\\","y","n","like","se","en","te","el","I'm"}));
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/twitter/src/main/java/com/datatorrent/demos/twitter/TwitterTopCounterApplication.java
----------------------------------------------------------------------
diff --git a/demos/twitter/src/main/java/com/datatorrent/demos/twitter/TwitterTopCounterApplication.java b/demos/twitter/src/main/java/com/datatorrent/demos/twitter/TwitterTopCounterApplication.java
deleted file mode 100644
index 731a38f..0000000
--- a/demos/twitter/src/main/java/com/datatorrent/demos/twitter/TwitterTopCounterApplication.java
+++ /dev/null
@@ -1,222 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.twitter;
-
-import java.net.URI;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-
-import com.google.common.collect.Maps;
-
-import com.datatorrent.api.Context;
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.DAG.Locality;
-import com.datatorrent.api.Operator;
-import com.datatorrent.api.Operator.OutputPort;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-
-import com.datatorrent.contrib.twitter.TwitterSampleInput;
-import com.datatorrent.lib.algo.UniqueCounter;
-import com.datatorrent.lib.appdata.schemas.SchemaUtils;
-import com.datatorrent.lib.appdata.snapshot.AppDataSnapshotServerMap;
-import com.datatorrent.lib.io.ConsoleOutputOperator;
-import com.datatorrent.lib.io.PubSubWebSocketAppDataQuery;
-import com.datatorrent.lib.io.PubSubWebSocketAppDataResult;
-
-/**
- * Twitter Demo Application: <br>
- * This demo application samples random public status from twitter, send to url
- * extractor. <br>
- * Top 10 url(s) mentioned in tweets in last 5 mins are displayed on every
- * window count (500ms).<br>
- * <br>
- *
- * Real Time Calculation :<br>
- * This application calculates top 10 url mentioned in tweets in last 5
- * minutes across a 1% random tweet sampling on a rolling window basis.<br>
- * <br>
- * Before running this application, you need to have a <a href="https://dev.twitter.com/apps">Twitter API account</a>
- * and configure the authentication. For launch from CLI, those go into ~/.dt/dt-site.xml:
- * <pre>
- * {@code
- * <?xml version="1.0" encoding="UTF-8"?>
- * <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
- * <configuration>
- *
- * <property> <name>dt.operator.TweetSampler.consumerKey</name>
- * <value>TBD</value> </property>
- *
- * <property> <name>dt.operator.TweetSampler.consumerSecret</name>
- * <value>TBD</value> </property>
- *
- * <property> <name>dt.operator.TweetSampler.accessToken</name>
- * <value>TBD</value> </property>
- *
- * <property> <name>dt.operator.TweetSampler.accessTokenSecret</name>
- * <value>TBD</value> </property>
- * </configuration>
- * }
- * </pre>
- * Custom Attributes: <br>
- * <b>topCounts operator : <b>
- * <ul>
- * <li>Top Count : 10, number of top unique url to be reported.</li>
- * <li>Sliding window count : 600, report over last 5 min (600 * .5 / 60 mins)</li>
- * <li>window slide value : 1</li>
- * </ul>
- * <p>
- * Running Java Test or Main app in IDE:
- *
- * <pre>
- * LocalMode.runApp(new Application(), 600000); // 10 min run
- * </pre>
- *
- * Run Success : <br>
- * For successful deployment and run, user should see following output on
- * console:
- *
- * <pre>
- * topURLs: {http://goo.gl/V0R05=2, http://etsy.me/10r1Yg3=6, http://tinyurl.com/88b5jqb=2, http://www.justunfollow.com=4, http://fllwrs.com=2, http://goo.gl/a9Sjp=2, http://goo.gl/iKeVH=2, http://Unfollowers.me=7, http://freetexthost.com/j3y03la4g3=2, http://uranaitter.com=4}
- * topURLs: {http://goo.gl/V0R05=2, http://etsy.me/10r1Yg3=6, http://tinyurl.com/88b5jqb=2, http://www.justunfollow.com=4, http://fllwrs.com=2, http://goo.gl/a9Sjp=2, http://goo.gl/iKeVH=2, http://Unfollowers.me=7, http://freetexthost.com/j3y03la4g3=2, http://uranaitter.com=4}
- * topURLs: {http://goo.gl/V0R05=2, http://etsy.me/10r1Yg3=6, http://tinyurl.com/88b5jqb=2, http://www.justunfollow.com=4, http://fllwrs.com=2, http://goo.gl/a9Sjp=2, http://goo.gl/iKeVH=2, http://Unfollowers.me=7, http://freetexthost.com/j3y03la4g3=2, http://uranaitter.com=4}
- * topURLs: {http://goo.gl/V0R05=2, http://etsy.me/10r1Yg3=6, http://tinyurl.com/88b5jqb=2, http://www.justunfollow.com=4, http://fllwrs.com=2, http://goo.gl/a9Sjp=2, http://goo.gl/iKeVH=2, http://Unfollowers.me=7, http://freetexthost.com/j3y03la4g3=2, http://uranaitter.com=4}
- * topURLs: {http://goo.gl/V0R05=2, http://etsy.me/10r1Yg3=6, http://tinyurl.com/88b5jqb=2, http://www.justunfollow.com=4, http://fllwrs.com=2, http://goo.gl/a9Sjp=2, http://goo.gl/iKeVH=2, http://Unfollowers.me=7, http://freetexthost.com/j3y03la4g3=2, http://uranaitter.com=4}
- * topURLs: {http://goo.gl/V0R05=2, http://etsy.me/10r1Yg3=6, http://tinyurl.com/88b5jqb=2, http://www.justunfollow.com=4, http://fllwrs.com=2, http://goo.gl/a9Sjp=2, http://goo.gl/iKeVH=2, http://Unfollowers.me=7, http://freetexthost.com/j3y03la4g3=2, http://uranaitter.com=4}
- * topURLs: {http://goo.gl/V0R05=2, http://etsy.me/10r1Yg3=6, http://tinyurl.com/88b5jqb=2, http://www.justunfollow.com=4, http://fllwrs.com=2, http://goo.gl/a9Sjp=2, http://goo.gl/iKeVH=2, http://Unfollowers.me=7, http://freetexthost.com/j3y03la4g3=2, http://uranaitter.com=4}
- * 2013-06-17 14:38:55,201 [main] INFO stram.StramLocalCluster run - Application finished.
- * 2013-06-17 14:38:55,201 [container-2] INFO stram.StramChild processHeartbeatResponse - Received shutdown request
- * </pre>
- *
- * Scaling Options : <br>
- * User can scale application by setting intial partition size > 1 on count
- * unique operator. <br>
- * <br>
- *
- * Application DAG : <br>
- * <img src="doc-files/Application.gif" width=600px > <br>
- * <br>
- *
- * Streaming Window Size : 500ms(default) <br>
- * Operator Details : <br>
- * <ul>
- * <li><b>The twitterFeed operator : </b> This operator samples random public
- * statues from twitter and emits to application. <br>
- * Class : com.datatorrent.demos.twitter.TwitterSampleInput <br>
- * StateFull : No, window count 1 <br>
- * </li>
- * <li><b>The urlExtractor operator : </b> This operator extracts url from
- * random sampled statues from twitter. <br>
- * Class : {@link com.datatorrent.demos.twitter.TwitterStatusURLExtractor} <br>
- * StateFull : No, window count 1 <br>
- * </li>
- * <li><b>The uniqueCounter operator : </b> This operator aggregates count for each
- * url extracted from random samples. <br>
- * Class : {@link com.datatorrent.lib.algo.UniqueCounter} <br>
- * StateFull : No, window count 1 <br>
- * </li>
- * <li><b> The topCounts operator : </b> This operator caluculates top url in last 1
- * min sliding window count 1. <br>
- * Class : com.datatorrent.lib.algo.WindowedTopCounter <br>
- * StateFull : Yes, sliding window count 120 (1 min) <br>
- * </li>
- * <li><b>The operator Console: </b> This operator just outputs the input tuples
- * to the console (or stdout). <br>
- * </li>
- * </ul>
- *
- * @since 0.3.2
- */
-@ApplicationAnnotation(name = TwitterTopCounterApplication.APP_NAME)
-public class TwitterTopCounterApplication implements StreamingApplication
-{
- public static final String SNAPSHOT_SCHEMA = "twitterURLDataSchema.json";
- public static final String CONVERSION_SCHEMA = "twitterURLConverterSchema.json";
- public static final String APP_NAME = "TwitterDemo";
-
- private final Locality locality = null;
-
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- // Setup the operator to get the data from twitter sample stream injected into the system.
- TwitterSampleInput twitterFeed = new TwitterSampleInput();
- twitterFeed = dag.addOperator("TweetSampler", twitterFeed);
-
- // Setup the operator to get the URLs extracted from the twitter statuses
- TwitterStatusURLExtractor urlExtractor = dag.addOperator("URLExtractor", TwitterStatusURLExtractor.class);
-
- // Setup a node to count the unique urls within a window.
- UniqueCounter<String> uniqueCounter = dag.addOperator("UniqueURLCounter", new UniqueCounter<String>());
- // Get the aggregated url counts and count them over last 5 mins.
- dag.setAttribute(uniqueCounter, Context.OperatorContext.APPLICATION_WINDOW_COUNT, 600);
- dag.setAttribute(uniqueCounter, Context.OperatorContext.SLIDE_BY_WINDOW_COUNT, 1);
-
-
- WindowedTopCounter<String> topCounts = dag.addOperator("TopCounter", new WindowedTopCounter<String>());
- topCounts.setTopCount(10);
- topCounts.setSlidingWindowWidth(1);
- topCounts.setDagWindowWidth(1);
-
- // Feed the statuses from feed into the input of the url extractor.
- dag.addStream("TweetStream", twitterFeed.status, urlExtractor.input).setLocality(Locality.CONTAINER_LOCAL);
- // Start counting the urls coming out of URL extractor
- dag.addStream("TwittedURLs", urlExtractor.url, uniqueCounter.data).setLocality(locality);
- // Count unique urls
- dag.addStream("UniqueURLCounts", uniqueCounter.count, topCounts.input);
-
- consoleOutput(dag, "topURLs", topCounts.output, SNAPSHOT_SCHEMA, "url");
- }
-
- public static void consoleOutput(DAG dag, String operatorName, OutputPort<List<Map<String, Object>>> topCount, String schemaFile, String alias)
- {
- String gatewayAddress = dag.getValue(DAG.GATEWAY_CONNECT_ADDRESS);
- if (!StringUtils.isEmpty(gatewayAddress)) {
- URI uri = URI.create("ws://" + gatewayAddress + "/pubsub");
-
- AppDataSnapshotServerMap snapshotServer = dag.addOperator("SnapshotServer", new AppDataSnapshotServerMap());
-
- Map<String, String> conversionMap = Maps.newHashMap();
- conversionMap.put(alias, WindowedTopCounter.FIELD_TYPE);
- String snapshotServerJSON = SchemaUtils.jarResourceFileToString(schemaFile);
-
- snapshotServer.setSnapshotSchemaJSON(snapshotServerJSON);
- snapshotServer.setTableFieldToMapField(conversionMap);
-
- PubSubWebSocketAppDataQuery wsQuery = new PubSubWebSocketAppDataQuery();
- wsQuery.setUri(uri);
- snapshotServer.setEmbeddableQueryInfoProvider(wsQuery);
-
- PubSubWebSocketAppDataResult wsResult = dag.addOperator("QueryResult", new PubSubWebSocketAppDataResult());
- wsResult.setUri(uri);
- Operator.InputPort<String> queryResultPort = wsResult.input;
-
- dag.addStream("MapProvider", topCount, snapshotServer.input);
- dag.addStream("Result", snapshotServer.queryResult, queryResultPort);
- } else {
- ConsoleOutputOperator operator = dag.addOperator(operatorName, new ConsoleOutputOperator());
- operator.setStringFormat(operatorName + ": %s");
-
- dag.addStream("MapProvider", topCount, operator.input);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/twitter/src/main/java/com/datatorrent/demos/twitter/TwitterTopWordsApplication.java
----------------------------------------------------------------------
diff --git a/demos/twitter/src/main/java/com/datatorrent/demos/twitter/TwitterTopWordsApplication.java b/demos/twitter/src/main/java/com/datatorrent/demos/twitter/TwitterTopWordsApplication.java
deleted file mode 100644
index 3953ab7..0000000
--- a/demos/twitter/src/main/java/com/datatorrent/demos/twitter/TwitterTopWordsApplication.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.twitter;
-
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.DAG.Locality;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-import com.datatorrent.contrib.twitter.TwitterSampleInput;
-import com.datatorrent.lib.algo.UniqueCounter;
-
-/**
- * This application is same as other twitter demo
- * {@link com.datatorrent.demos.twitter.TwitterTopCounterApplication} <br>
- * Run Sample :
- *
- * <pre>
- * 2013-06-17 16:50:34,911 [Twitter Stream consumer-1[Establishing connection]] INFO twitter4j.TwitterStreamImpl info - Connection established.
- * 2013-06-17 16:50:34,912 [Twitter Stream consumer-1[Establishing connection]] INFO twitter4j.TwitterStreamImpl info - Receiving status stream.
- * topWords: {}
- * topWords: {love=1, ate=1, catch=1, calma=1, Phillies=1, ela=1, from=1, running=1}
- * </pre>
- *
- * @since 0.3.2
- */
-@ApplicationAnnotation(name = TwitterTopWordsApplication.APP_NAME)
-public class TwitterTopWordsApplication implements StreamingApplication
-{
- public static final String SNAPSHOT_SCHEMA = "twitterWordDataSchema.json";
- public static final String CONVERSION_SCHEMA = "twitterWordConverterSchema.json";
- public static final String APP_NAME = "RollingTopWordsDemo";
- public static final String PROP_USE_APPDATA = "dt.application." + APP_NAME + ".useAppData";
-
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- TwitterSampleInput twitterFeed = new TwitterSampleInput();
- twitterFeed = dag.addOperator("TweetSampler", twitterFeed);
-
- TwitterStatusWordExtractor wordExtractor = dag.addOperator("WordExtractor", TwitterStatusWordExtractor.class);
- UniqueCounter<String> uniqueCounter = dag.addOperator("UniqueWordCounter", new UniqueCounter<String>());
- WindowedTopCounter<String> topCounts = dag.addOperator("TopCounter", new WindowedTopCounter<String>());
-
- topCounts.setSlidingWindowWidth(120);
- topCounts.setDagWindowWidth(1);
-
- dag.addStream("TweetStream", twitterFeed.text, wordExtractor.input);
- dag.addStream("TwittedWords", wordExtractor.output, uniqueCounter.data);
- dag.addStream("UniqueWordCounts", uniqueCounter.count, topCounts.input).setLocality(Locality.CONTAINER_LOCAL);
-
- TwitterTopCounterApplication.consoleOutput(dag, "topWords", topCounts.output, SNAPSHOT_SCHEMA, "word");
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/twitter/src/main/java/com/datatorrent/demos/twitter/TwitterTrendingHashtagsApplication.java
----------------------------------------------------------------------
diff --git a/demos/twitter/src/main/java/com/datatorrent/demos/twitter/TwitterTrendingHashtagsApplication.java b/demos/twitter/src/main/java/com/datatorrent/demos/twitter/TwitterTrendingHashtagsApplication.java
deleted file mode 100644
index 3597a92..0000000
--- a/demos/twitter/src/main/java/com/datatorrent/demos/twitter/TwitterTrendingHashtagsApplication.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.twitter;
-
-import org.apache.hadoop.conf.Configuration;
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.DAG.Locality;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-import com.datatorrent.contrib.twitter.TwitterSampleInput;
-import com.datatorrent.lib.algo.UniqueCounter;
-
-/**
- * Twitter Demo Application: <br>
- * This demo application samples random public status from twitter, send to Hashtag
- * extractor. <br>
- * Top 10 Hashtag(s) mentioned in tweets in last 5 mins are displayed on every
- * window count (500ms).<br>
- * <br>
- *
- * Real Time Calculation :<br>
- * This application calculates top 10 Hashtag mentioned in tweets in last 5
- * minutes across a 1% random tweet sampling on a rolling window basis.<br>
- * <br>
- * Before running this application, you need to have a <a href="https://dev.twitter.com/apps">Twitter API account</a>
- * and configure the authentication. For launch from CLI, those go into ~/.dt/dt-site.xml:
- * <pre>
- * {@code
- * <?xml version="1.0" encoding="UTF-8"?>
- * <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
- * <configuration>
- *
- * <property> <name>dt.operator.TweetSampler.prop.consumerKey</name>
- * <value>TBD</value> </property>
- *
- * <property> <name>dt.operator.TweetSampler.prop.consumerSecret</name>
- * <value>TBD</value> </property>
- *
- * <property> <name>dt.operator.TweetSampler.prop.accessToken</name>
- * <value>TBD</value> </property>
- *
- * <property> <name>dt.operator.TweetSampler.prop.accessTokenSecret</name>
- * <value>TBD</value> </property>
- * </configuration>
- * }
- * </pre>
- * Custom Attributes: <br>
- * <b>topCounts operator : <b>
- * <ul>
- * <li>Top Count : 10, number of top unique Hashtag to be reported.</li>
- * <li>Sliding window count : 600, report over last 5 min (600 * .5 / 60 mins)</li>
- * <li>window slide value : 1</li>
- * </ul>
- * <p>
- * Running Java Test or Main app in IDE:
- *
- * <pre>
- * LocalMode.runApp(new Application(), 600000); // 10 min run
- * </pre>
- *
- * Run Success : <br>
- * For successful deployment and run, user should see similar output on console as below:
- *
- * <pre>
- * topHashtags: {"string": "{\"gameinsight\":\"12\",\"android\":\"8\",\"WotaJKT48alay\":\"12\",\"TernyataJKT48CyberItuForoumSampah\":\"8\",\"NHL15Subban\":\"30\",\"JKT48CYBERKeracunanCUKRIK\":\"8\",\"verifydjzoodel\":\"59\",\"androidgames\":\"9\",\"\u0645\u0639_\u0627\u0644\u0644\u0647\":\"10\",\"Jct48cyberTAIBABI\":\"10\"}"}
- * topHashtags: {"string": "{\"gameinsight\":\"12\",\"android\":\"8\",\"WotaJKT48alay\":\"12\",\"TernyataJKT48CyberItuForoumSampah\":\"8\",\"NHL15Subban\":\"30\",\"JKT48CYBERKeracunanCUKRIK\":\"8\",\"verifydjzoodel\":\"59\",\"androidgames\":\"9\",\"\u0645\u0639_\u0627\u0644\u0644\u0647\":\"10\",\"Jct48cyberTAIBABI\":\"10\"}"}
- * topHashtags: {"string": "{\"gameinsight\":\"12\",\"android\":\"8\",\"WotaJKT48alay\":\"12\",\"TernyataJKT48CyberItuForoumSampah\":\"8\",\"NHL15Subban\":\"30\",\"JKT48CYBERKeracunanCUKRIK\":\"8\",\"verifydjzoodel\":\"59\",\"androidgames\":\"9\",\"\u0645\u0639_\u0627\u0644\u0644\u0647\":\"10\",\"Jct48cyberTAIBABI\":\"10\"}"}
- * topHashtags: {"string": "{\"gameinsight\":\"12\",\"android\":\"8\",\"WotaJKT48alay\":\"12\",\"TernyataJKT48CyberItuForoumSampah\":\"8\",\"NHL15Subban\":\"30\",\"JKT48CYBERKeracunanCUKRIK\":\"8\",\"verifydjzoodel\":\"59\",\"androidgames\":\"9\",\"\u0645\u0639_\u0627\u0644\u0644\u0647\":\"10\",\"Jct48cyberTAIBABI\":\"10\"}"}
- * topHashtags: {"string": "{\"gameinsight\":\"12\",\"android\":\"8\",\"WotaJKT48alay\":\"12\",\"TernyataJKT48CyberItuForoumSampah\":\"8\",\"NHL15Subban\":\"30\",\"JKT48CYBERKeracunanCUKRIK\":\"8\",\"verifydjzoodel\":\"59\",\"androidgames\":\"9\",\"\u0645\u0639_\u0627\u0644\u0644\u0647\":\"10\",\"Jct48cyberTAIBABI\":\"10\"}"}
- * topHashtags: {"string": "{\"gameinsight\":\"12\",\"android\":\"8\",\"WotaJKT48alay\":\"12\",\"TernyataJKT48CyberItuForoumSampah\":\"8\",\"NHL15Subban\":\"30\",\"JKT48CYBERKeracunanCUKRIK\":\"8\",\"verifydjzoodel\":\"59\",\"androidgames\":\"9\",\"\u0645\u0639_\u0627\u0644\u0644\u0647\":\"10\",\"Jct48cyberTAIBABI\":\"10\"}"}
- * topHashtags: {"string": "{\"gameinsight\":\"12\",\"android\":\"8\",\"WotaJKT48alay\":\"12\",\"TernyataJKT48CyberItuForoumSampah\":\"8\",\"NHL15Subban\":\"30\",\"JKT48CYBERKeracunanCUKRIK\":\"8\",\"verifydjzoodel\":\"59\",\"androidgames\":\"9\",\"\u0645\u0639_\u0627\u0644\u0644\u0647\":\"10\",\"Jct48cyberTAIBABI\":\"10\"}"}
- * topHashtags: {"string": "{\"gameinsight\":\"12\",\"android\":\"8\",\"WotaJKT48alay\":\"12\",\"TernyataJKT48CyberItuForoumSampah\":\"8\",\"NHL15Subban\":\"30\",\"JKT48CYBERKeracunanCUKRIK\":\"8\",\"verifydjzoodel\":\"59\",\"androidgames\":\"9\",\"\u0645\u0639_\u0627\u0644\u0644\u0647\":\"10\",\"Jct48cyberTAIBABI\":\"10\"}"}
- * topHashtags: {"string": "{\"gameinsight\":\"12\",\"android\":\"8\",\"WotaJKT48alay\":\"12\",\"TernyataJKT48CyberItuForoumSampah\":\"8\",\"NHL15Subban\":\"30\",\"JKT48CYBERKeracunanCUKRIK\":\"8\",\"verifydjzoodel\":\"59\",\"androidgames\":\"9\",\"\u0645\u0639_\u0627\u0644\u0644\u0647\":\"10\",\"Jct48cyberTAIBABI\":\"10\"}"}
- * topHashtags: {"string": "{\"gameinsight\":\"12\",\"android\":\"8\",\"WotaJKT48alay\":\"12\",\"TernyataJKT48CyberItuForoumSampah\":\"8\",\"NHL15Subban\":\"30\",\"JKT48CYBERKeracunanCUKRIK\":\"8\",\"verifydjzoodel\":\"59\",\"androidgames\":\"9\",\"\u0645\u0639_\u0627\u0644\u0644\u0647\":\"10\",\"Jct48cyberTAIBABI\":\"10\"}"}
- * 2013-06-17 14:38:55,201 [main] INFO stram.StramLocalCluster run - Application finished.
- * 2013-06-17 14:38:55,201 [container-2] INFO stram.StramChild processHeartbeatResponse - Received shutdown request
- * </pre>
- *
- * Scaling Options : <br>
- * User can scale application by setting intial partition size > 1 on count
- * unique operator. <br>
- * <br>
- *
- * Application DAG : <br>
- * <img src="doc-files/Application.gif" width=600px > <br>
- * <br>
- *
- * Streaming Window Size : 500ms(default) <br>
- * Operator Details : <br>
- * <ul>
- * <li><b>The twitterFeed operator : </b> This operator samples random public
- * statues from twitter and emits to application. <br>
- * Class : com.datatorrent.demos.twitter.TwitterSampleInput <br>
- * StateFull : No, window count 1 <br>
- * </li>
- * <li><b>The HashtagExtractor operator : </b> This operator extracts Hashtag from
- * random sampled statues from twitter. <br>
- * Class : {@link com.datatorrent.demos.twitter.TwitterStatusHashtagExtractor} <br>
- * StateFull : No, window count 1 <br>
- * </li>
- * <li><b>The uniqueCounter operator : </b> This operator aggregates count for each
- * Hashtag extracted from random samples. <br>
- * Class : {@link com.datatorrent.lib.algo.UniqueCounter} <br>
- * StateFull : No, window count 1 <br>
- * </li>
- * <li><b> The topCounts operator : </b> This operator caluculates top Hashtag in last 1
- * min sliding window count 1. <br>
- * Class : com.datatorrent.lib.algo.WindowedTopCounter <br>
- * StateFull : Yes, sliding window count 120 (1 min) <br>
- * </li>
- * <li><b>The operator Console: </b> This operator just outputs the input tuples
- * to the console (or stdout). <br>
- * </li>
- * </ul>
- *
- * @since 1.0.2
- */
-@ApplicationAnnotation(name = TwitterTrendingHashtagsApplication.APP_NAME)
-public class TwitterTrendingHashtagsApplication implements StreamingApplication
-{
- public static final String SNAPSHOT_SCHEMA = "twitterHashTagDataSchema.json";
- public static final String CONVERSION_SCHEMA = "twitterHashTagConverterSchema.json";
- public static final String APP_NAME = "TwitterTrendingDemo";
- public static final String PROP_USE_APPDATA = "dt.application." + APP_NAME + ".useAppData";
-
- private final Locality locality = null;
-
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- // Setup the operator to get the data from twitter sample stream injected into the system.
- TwitterSampleInput twitterFeed = new TwitterSampleInput();
- twitterFeed = dag.addOperator("TweetSampler", twitterFeed);
-
- // Setup a node to count the unique Hashtags within a window.
- UniqueCounter<String> uniqueCounter = dag.addOperator("UniqueHashtagCounter", new UniqueCounter<String>());
-
- // Get the aggregated Hashtag counts and count them over last 5 mins.
- WindowedTopCounter<String> topCounts = dag.addOperator("TopCounter", new WindowedTopCounter<String>());
- topCounts.setTopCount(10);
- topCounts.setSlidingWindowWidth(600);
- topCounts.setDagWindowWidth(1);
-
- dag.addStream("TwittedHashtags", twitterFeed.hashtag, uniqueCounter.data).setLocality(locality);
- // Count unique Hashtags
- dag.addStream("UniqueHashtagCounts", uniqueCounter.count, topCounts.input);
-
- TwitterTopCounterApplication.consoleOutput(dag, "topHashtags", topCounts.output, SNAPSHOT_SCHEMA, "hashtag");
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/twitter/src/main/java/com/datatorrent/demos/twitter/URLSerDe.java
----------------------------------------------------------------------
diff --git a/demos/twitter/src/main/java/com/datatorrent/demos/twitter/URLSerDe.java b/demos/twitter/src/main/java/com/datatorrent/demos/twitter/URLSerDe.java
deleted file mode 100644
index 43ed8f7..0000000
--- a/demos/twitter/src/main/java/com/datatorrent/demos/twitter/URLSerDe.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.twitter;
-
-import java.nio.ByteBuffer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import com.datatorrent.api.StreamCodec;
-import com.datatorrent.netlet.util.Slice;
-
-/**
- * <p>URLSerDe class.</p>
- *
- * @since 0.3.2
- */
-public class URLSerDe implements StreamCodec<byte[]>
-{
- /**
- * Covert the bytes into object useful for downstream node.
- *
- * @param fragment
- * @return WindowedURLHolder object which represents the bytes.
- */
- @Override
- public byte[] fromByteArray(Slice fragment)
- {
- if (fragment == null || fragment.buffer == null) {
- return null;
- } else if (fragment.offset == 0 && fragment.length == fragment.buffer.length) {
- return fragment.buffer;
- } else {
- byte[] buffer = new byte[fragment.buffer.length];
- System.arraycopy(fragment.buffer, fragment.offset, buffer, 0, fragment.length);
- return buffer;
- }
- }
-
- /**
- * Cast the input object to byte[].
- *
- * @param object - byte array representing the bytes of the string
- * @return the same object as input
- */
- @Override
- public Slice toByteArray(byte[] object)
- {
- return new Slice(object, 0, object.length);
- }
-
- @Override
- public int getPartition(byte[] object)
- {
- ByteBuffer bb = ByteBuffer.wrap(object);
- return bb.hashCode();
- }
-
- private static final Logger logger = LoggerFactory.getLogger(URLSerDe.class);
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/twitter/src/main/java/com/datatorrent/demos/twitter/WindowedTopCounter.java
----------------------------------------------------------------------
diff --git a/demos/twitter/src/main/java/com/datatorrent/demos/twitter/WindowedTopCounter.java b/demos/twitter/src/main/java/com/datatorrent/demos/twitter/WindowedTopCounter.java
deleted file mode 100644
index 20bb673..0000000
--- a/demos/twitter/src/main/java/com/datatorrent/demos/twitter/WindowedTopCounter.java
+++ /dev/null
@@ -1,282 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.twitter;
-
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.PriorityQueue;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-
-import com.datatorrent.api.Context.OperatorContext;
-import com.datatorrent.api.DefaultInputPort;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.common.util.BaseOperator;
-
-/**
- *
- * WindowedTopCounter is an operator which counts the most often occurring tuples in a sliding window of a specific size.
- * The operator expects to receive a map object which contains a set of objects mapped to their respective frequency of
- * occurrences. e.g. if we are looking at most commonly occurring names then the operator expects to receive the tuples
- * of type Map<String, Intenger> on its input port, and at the end of the window it emits 1 object of type Map<String, Integer>
- * with a pre determined size. The emitted object contains the most frequently occurring keys.
- *
- * @param <T> Type of the key in the map object which is accepted on input port as payload. Note that this key must be HashMap friendly.
- * @since 0.3.2
- */
-public class WindowedTopCounter<T> extends BaseOperator
-{
- public static final String FIELD_TYPE = "type";
- public static final String FIELD_COUNT = "count";
-
- private static final Logger logger = LoggerFactory.getLogger(WindowedTopCounter.class);
-
- private PriorityQueue<SlidingContainer<T>> topCounter;
- private int windows;
- private int topCount = 10;
- private int slidingWindowWidth;
- private int dagWindowWidth;
- private HashMap<T, SlidingContainer<T>> objects = new HashMap<T, SlidingContainer<T>>();
-
- /**
- * Input port on which map objects containing keys with their respective frequency as values will be accepted.
- */
- public final transient DefaultInputPort<Map<T, Integer>> input = new DefaultInputPort<Map<T, Integer>>()
- {
- @Override
- public void process(Map<T, Integer> map)
- {
- for (Map.Entry<T, Integer> e : map.entrySet()) {
- SlidingContainer<T> holder = objects.get(e.getKey());
- if (holder == null) {
- holder = new SlidingContainer<T>(e.getKey(), windows);
- objects.put(e.getKey(), holder);
- }
- holder.adjustCount(e.getValue());
- }
- }
- };
-
- public final transient DefaultOutputPort<List<Map<String, Object>>> output = new DefaultOutputPort<List<Map<String, Object>>>();
-
- @Override
- public void setup(OperatorContext context)
- {
- windows = (int)(slidingWindowWidth / dagWindowWidth) + 1;
- if (slidingWindowWidth % dagWindowWidth != 0) {
- logger.warn("slidingWindowWidth(" + slidingWindowWidth + ") is not exact multiple of dagWindowWidth(" + dagWindowWidth + ")");
- }
-
- topCounter = new PriorityQueue<SlidingContainer<T>>(this.topCount, new TopSpotComparator());
- }
-
- @Override
- public void beginWindow(long windowId)
- {
- topCounter.clear();
- }
-
- @Override
- public void endWindow()
- {
- Iterator<Map.Entry<T, SlidingContainer<T>>> iterator = objects.entrySet().iterator();
- int i = topCount;
-
- /*
- * Try to fill the priority queue with the first topCount URLs.
- */
- SlidingContainer<T> holder;
- while (iterator.hasNext()) {
- holder = iterator.next().getValue();
- holder.slide();
-
- if (holder.totalCount == 0) {
- iterator.remove();
- } else {
- topCounter.add(holder);
- if (--i == 0) {
- break;
- }
- }
- }
- logger.debug("objects.size(): {}", objects.size());
-
- /*
- * Make room for the new element in the priority queue by deleting the
- * smallest one, if we KNOW that the new element is useful to us.
- */
- if (i == 0) {
- int smallest = topCounter.peek().totalCount;
- while (iterator.hasNext()) {
- holder = iterator.next().getValue();
- holder.slide();
-
- if (holder.totalCount > smallest) {
- topCounter.poll();
- topCounter.add(holder);
- smallest = topCounter.peek().totalCount;
- } else if (holder.totalCount == 0) {
- iterator.remove();
- }
- }
- }
-
- List<Map<String, Object>> data = Lists.newArrayList();
-
- Iterator<SlidingContainer<T>> topIter = topCounter.iterator();
-
- while (topIter.hasNext()) {
- final SlidingContainer<T> wh = topIter.next();
- Map<String, Object> tableRow = Maps.newHashMap();
-
- tableRow.put(FIELD_TYPE, wh.identifier.toString());
- tableRow.put(FIELD_COUNT, wh.totalCount);
-
- data.add(tableRow);
- }
-
- Collections.sort(data, TwitterOutputSorter.INSTANCE);
-
- output.emit(data);
- topCounter.clear();
- }
-
- @Override
- public void teardown()
- {
- topCounter = null;
- objects = null;
- }
-
- /**
- * Set the count of most frequently occurring keys to emit per map object.
- *
- * @param count count of the objects in the map emitted at the output port.
- */
- public void setTopCount(int count)
- {
- topCount = count;
- }
-
- public int getTopCount()
- {
- return topCount;
- }
-
- /**
- * @return the windows
- */
- public int getWindows()
- {
- return windows;
- }
-
- /**
- * @param windows the windows to set
- */
- public void setWindows(int windows)
- {
- this.windows = windows;
- }
-
- /**
- * @return the slidingWindowWidth
- */
- public int getSlidingWindowWidth()
- {
- return slidingWindowWidth;
- }
-
- /**
- * Set the width of the sliding window.
- *
- * Sliding window is typically much larger than the dag window. e.g. One may want to measure the most frequently
- * occurring keys over the period of 5 minutes. So if dagWindowWidth (which is by default 500ms) is set to 500ms,
- * the slidingWindowWidth would be (60 * 5 * 1000 =) 300000.
- *
- * @param slidingWindowWidth - Sliding window width to be set for this operator, recommended to be multiple of DAG window.
- */
- public void setSlidingWindowWidth(int slidingWindowWidth)
- {
- this.slidingWindowWidth = slidingWindowWidth;
- }
-
- /**
- * @return the dagWindowWidth
- */
- public int getDagWindowWidth()
- {
- return dagWindowWidth;
- }
-
- /**
- * Set the width of the sliding window.
- *
- * Sliding window is typically much larger than the dag window. e.g. One may want to measure the most frequently
- * occurring keys over the period of 5 minutes. So if dagWindowWidth (which is by default 500ms) is set to 500ms,
- * the slidingWindowWidth would be (60 * 5 * 1000 =) 300000.
- *
- * @param dagWindowWidth - DAG's native window width. It has to be the value of the native window set at the application level.
- */
- public void setDagWindowWidth(int dagWindowWidth)
- {
- this.dagWindowWidth = dagWindowWidth;
- }
-
- static class TopSpotComparator implements Comparator<SlidingContainer<?>>
- {
- @Override
- public int compare(SlidingContainer<?> o1, SlidingContainer<?> o2)
- {
- if (o1.totalCount > o2.totalCount) {
- return 1;
- } else if (o1.totalCount < o2.totalCount) {
- return -1;
- }
-
- return 0;
- }
- }
-
- private static class TwitterOutputSorter implements Comparator<Map<String, Object>>
- {
- public static final TwitterOutputSorter INSTANCE = new TwitterOutputSorter();
-
- private TwitterOutputSorter()
- {
- }
-
- @Override
- public int compare(Map<String, Object> o1, Map<String, Object> o2)
- {
- Integer count1 = (Integer)o1.get(FIELD_COUNT);
- Integer count2 = (Integer)o2.get(FIELD_COUNT);
-
- return count1.compareTo(count2);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/twitter/src/main/java/com/datatorrent/demos/twitter/doc-files/Application.gif
----------------------------------------------------------------------
diff --git a/demos/twitter/src/main/java/com/datatorrent/demos/twitter/doc-files/Application.gif b/demos/twitter/src/main/java/com/datatorrent/demos/twitter/doc-files/Application.gif
deleted file mode 100644
index d21e1d9..0000000
Binary files a/demos/twitter/src/main/java/com/datatorrent/demos/twitter/doc-files/Application.gif and /dev/null differ
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/twitter/src/main/java/com/datatorrent/demos/twitter/package-info.java
----------------------------------------------------------------------
diff --git a/demos/twitter/src/main/java/com/datatorrent/demos/twitter/package-info.java b/demos/twitter/src/main/java/com/datatorrent/demos/twitter/package-info.java
deleted file mode 100644
index 5a02e4b..0000000
--- a/demos/twitter/src/main/java/com/datatorrent/demos/twitter/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/**
- * Twitter top URL's demonstration application.
- */
-package com.datatorrent.demos.twitter;
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/twitter/src/main/resources/META-INF/properties-TwitterKinesisDemo.xml
----------------------------------------------------------------------
diff --git a/demos/twitter/src/main/resources/META-INF/properties-TwitterKinesisDemo.xml b/demos/twitter/src/main/resources/META-INF/properties-TwitterKinesisDemo.xml
deleted file mode 100644
index 7d45153..0000000
--- a/demos/twitter/src/main/resources/META-INF/properties-TwitterKinesisDemo.xml
+++ /dev/null
@@ -1,52 +0,0 @@
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-
-<!-- properties for the twitter kinesis demo -->
-<configuration>
-
- <property>
- <name>dt.operator.FromKinesis.streamName</name>
- <value>TwitterTag</value>
- </property>
- <property>
- <name>dt.operator.FromKinesis.accessKey</name>
- </property>
- <property>
- <name>dt.operator.FromKinesis.secretKey</name>
- </property>
- <property>
- <name>dt.operator.FromKinesis.endPoint</name>
- </property>
- <property>
- <name>dt.operator.ToKinesis.streamName</name>
- <value>TwitterTag</value>
- </property>
- <property>
- <name>dt.operator.ToKinesis.accessKey</name>
- </property>
- <property>
- <name>dt.operator.ToKinesis.secretKey</name>
- </property>
- <property>
- <name>dt.operator.ToKinesis.endPoint</name>
- </property>
-
-</configuration>
[06/30] apex-malhar git commit: Renamed demos to examples. Packages
and artifactid names are changed as suggested.
Posted by th...@apache.org.
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/mrmonitor/src/main/java/org/apache/apex/examples/mrmonitor/MRStatusObject.java
----------------------------------------------------------------------
diff --git a/examples/mrmonitor/src/main/java/org/apache/apex/examples/mrmonitor/MRStatusObject.java b/examples/mrmonitor/src/main/java/org/apache/apex/examples/mrmonitor/MRStatusObject.java
new file mode 100644
index 0000000..ae8b551
--- /dev/null
+++ b/examples/mrmonitor/src/main/java/org/apache/apex/examples/mrmonitor/MRStatusObject.java
@@ -0,0 +1,501 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.mrmonitor;
+
+import java.util.LinkedList;
+import java.util.Map;
+import java.util.Queue;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * <p>
+ * MRStatusObject class.
+ * </p>
+ *
+ * @since 0.3.4
+ */
+public class MRStatusObject
+{
+ private String command;
+ /**
+ * This stores the Resource Manager/ Task Manager's host information
+ */
+ private String uri;
+ /**
+ * This field stores the job id
+ */
+ private String jobId;
+ /**
+ * This field stores the api version of the rest apis
+ */
+ private String apiVersion;
+ /**
+ * This field stores the hadoop version 1 for 1.x and 2 for 2.x
+ */
+ private int hadoopVersion;
+ /**
+ * This field stores the app id for the hadoop 2.x
+ */
+ private String appId;
+ /**
+ * This field stores the RM port information for hadoop 2.x / Task Manager server port for hadoop 1.X from where we
+ * can get the job information
+ */
+ private int rmPort;
+ /**
+ * This field stores the history server information for hadoop 2.x from where we can get the job information
+ */
+ private int historyServerPort;
+ /**
+ * This field stores the job information as json object
+ */
+ private JSONObject jsonObject;
+ /**
+ * This field tells if the object has been modified
+ */
+ private boolean modified;
+ /**
+ * This stores the mapping of map task ids to the TaskObject
+ */
+ private Map<String, TaskObject> mapJsonObject;
+ /**
+ * This stores the mapping of reduce task ids to the TaskObject
+ */
+ private Map<String, TaskObject> reduceJsonObject;
+ /**
+ * This holds the information about the various metrics like MAP_OUTPUT_RECORDS etc for this job
+ */
+ private TaskObject metricObject;
+
+ /**
+ * This holds the number of windows occurred when the new data was retrieved for this job
+ */
+ private int retrials;
+
+ /**
+ * The scheduler is used to store the job status every 1 minute
+ */
+ private transient ScheduledExecutorService statusScheduler;
+
+ /**
+ * This stores the progress of the map tasks
+ */
+ Queue<String> mapStatusHistory;
+
+ /**
+ * This stores the progress of the reduce tasks
+ */
+ Queue<String> reduceStatusHistory;
+
+ /**
+ * This stores the history of the physical memory usage
+ */
+ Queue<String> physicalMemoryStatusHistory;
+
+ /**
+ * This stores the history of the virtual memory usage
+ */
+ Queue<String> virtualMemoryStatusHistory;
+
+ /**
+ * This stores the history of the cpu
+ */
+ Queue<String> cpuStatusHistory;
+
+ /**
+ * The number of minutes for which the status history of map and reduce tasks is stored
+ */
+ private int statusHistoryCount = 60;
+
+ /**
+ * This field notifies if history status queues have changed over time
+ */
+ private boolean changedHistoryStatus;
+
+ public MRStatusObject()
+ {
+ retrials = 0;
+ modified = true;
+ mapJsonObject = new ConcurrentHashMap<String, TaskObject>();
+ reduceJsonObject = new ConcurrentHashMap<String, TaskObject>();
+ mapStatusHistory = new LinkedList<String>();
+ reduceStatusHistory = new LinkedList<String>();
+ physicalMemoryStatusHistory = new LinkedList<String>();
+ virtualMemoryStatusHistory = new LinkedList<String>();
+ cpuStatusHistory = new LinkedList<String>();
+ statusScheduler = Executors.newScheduledThreadPool(1);
+ statusScheduler.scheduleAtFixedRate(new Runnable()
+ {
+ @Override
+ public void run()
+ {
+ if (jsonObject != null) {
+ changedHistoryStatus = true;
+ if (mapStatusHistory.size() > statusHistoryCount) {
+ mapStatusHistory.poll();
+ reduceStatusHistory.poll();
+ physicalMemoryStatusHistory.poll();
+ virtualMemoryStatusHistory.poll();
+ cpuStatusHistory.poll();
+ }
+ if (hadoopVersion == 2) {
+ try {
+ mapStatusHistory.add(jsonObject.getJSONObject("job").getString("mapProgress"));
+ reduceStatusHistory.add(jsonObject.getJSONObject("job").getString("reduceProgress"));
+ if (metricObject.getJson() != null) {
+ JSONArray arr = metricObject.getJson().getJSONObject("jobCounters").getJSONArray("counterGroup");
+ int length = arr.length();
+ for (int i = 0; i < length; i++) {
+ if (arr.getJSONObject(i).get("counterGroupName").equals("org.apache.hadoop.mapreduce.TaskCounter")) {
+ JSONArray counters = arr.getJSONObject(i).getJSONArray("counter");
+ for (int j = 0; j < counters.length(); j++) {
+ JSONObject counterObj = counters.getJSONObject(j);
+ if (counterObj.get("name").equals("PHYSICAL_MEMORY_BYTES")) {
+ physicalMemoryStatusHistory.add(counterObj.getString("totalCounterValue"));
+ } else if (counterObj.get("name").equals("VIRTUAL_MEMORY_BYTES")) {
+ virtualMemoryStatusHistory.add(counterObj.getString("totalCounterValue"));
+ } else if (counterObj.get("name").equals("CPU_MILLISECONDS")) {
+ cpuStatusHistory.add(counterObj.getString("totalCounterValue"));
+ }
+ }
+ break;
+ }
+ }
+ }
+ } catch (JSONException e) {
+ logger.error("error setting status history {}", e.getMessage());
+ }
+ } else {
+ try {
+ mapStatusHistory.add(jsonObject.getJSONObject("mapTaskSummary").getString("progressPercentage"));
+ reduceStatusHistory.add(jsonObject.getJSONObject("reduceTaskSummary").getString("progressPercentage"));
+ JSONArray arr = jsonObject.getJSONArray("jobCounterGroupsInfo");
+ int length = arr.length();
+ for (int i = 0; i < length; i++) {
+ if (arr.getJSONObject(i).get("groupName").equals("Map-Reduce Framework")) {
+ JSONArray counters = arr.getJSONObject(i).getJSONArray("jobCountersInfo");
+ for (int j = 0; j < counters.length(); j++) {
+ JSONObject counterObj = counters.getJSONObject(j);
+ if (counterObj.get("name").equals("Physical memory (bytes) snapshot")) {
+ physicalMemoryStatusHistory.add(counterObj.getString("totalValue"));
+ } else if (counterObj.get("name").equals("Virtual memory (bytes) snapshot")) {
+ virtualMemoryStatusHistory.add(counterObj.getString("totalValue"));
+ } else if (counterObj.get("name").equals("CPU time spent (ms)")) {
+ cpuStatusHistory.add(counterObj.getString("totalValue"));
+ }
+ }
+ break;
+ }
+ }
+ } catch (JSONException e) {
+ logger.error("error setting status history {}", e.getMessage());
+ }
+ }
+ }
+ }
+ }, 0, 1, TimeUnit.MINUTES);
+ }
+
+ public Map<String, TaskObject> getMapJsonObject()
+ {
+ return mapJsonObject;
+ }
+
+ public void setMapJsonObject(Map<String, TaskObject> mapJsonObject)
+ {
+ this.mapJsonObject = mapJsonObject;
+ }
+
+ public Map<String, TaskObject> getReduceJsonObject()
+ {
+ return reduceJsonObject;
+ }
+
+ public void setReduceJsonObject(Map<String, TaskObject> reduceJsonObject)
+ {
+ this.reduceJsonObject = reduceJsonObject;
+ }
+
+ public String getUri()
+ {
+ return uri;
+ }
+
+ public void setUri(String uri)
+ {
+ this.uri = uri;
+ }
+
+ public String getJobId()
+ {
+ return jobId;
+ }
+
+ public void setJobId(String jobId)
+ {
+ this.jobId = jobId;
+ }
+
+ public String getApiVersion()
+ {
+ return apiVersion;
+ }
+
+ public void setApiVersion(String apiVersion)
+ {
+ this.apiVersion = apiVersion;
+ }
+
+ public int getHadoopVersion()
+ {
+ return hadoopVersion;
+ }
+
+ public void setHadoopVersion(int hadoopVersion)
+ {
+ this.hadoopVersion = hadoopVersion;
+ }
+
+ public String getAppId()
+ {
+ return appId;
+ }
+
+ public void setAppId(String appId)
+ {
+ this.appId = appId;
+ }
+
+ public int getRmPort()
+ {
+ return rmPort;
+ }
+
+ public void setRmPort(int rmPort)
+ {
+ this.rmPort = rmPort;
+ }
+
+ public int getHistoryServerPort()
+ {
+ return historyServerPort;
+ }
+
+ public void setHistoryServerPort(int historyServerPort)
+ {
+ this.historyServerPort = historyServerPort;
+ }
+
+ public JSONObject getJsonObject()
+ {
+ return jsonObject;
+ }
+
+ public void setJsonObject(JSONObject jsonObject)
+ {
+ this.jsonObject = jsonObject;
+ }
+
+ public boolean isChangedHistoryStatus()
+ {
+ return changedHistoryStatus;
+ }
+
+ public void setChangedHistoryStatus(boolean changedHistoryStatus)
+ {
+ this.changedHistoryStatus = changedHistoryStatus;
+ }
+
+ @Override
+ public boolean equals(Object that)
+ {
+ if (this == that) {
+ return true;
+ }
+ if (!(that instanceof MRStatusObject)) {
+ return false;
+ }
+ if (this.hashCode() == that.hashCode()) {
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return (uri + jobId + apiVersion + String.valueOf(hadoopVersion)).hashCode();
+
+ }
+
+ public String getCommand()
+ {
+ return command;
+ }
+
+ public void setCommand(String command)
+ {
+ this.command = command;
+ }
+
+ public boolean isModified()
+ {
+ return modified;
+ }
+
+ public void setModified(boolean modified)
+ {
+ this.modified = modified;
+ }
+
+ public int getRetrials()
+ {
+ return retrials;
+ }
+
+ public void setRetrials(int retrials)
+ {
+ this.retrials = retrials;
+ }
+
+ public TaskObject getMetricObject()
+ {
+ return metricObject;
+ }
+
+ public void setMetricObject(TaskObject metricObject)
+ {
+ this.metricObject = metricObject;
+ }
+
+ public int getStatusHistoryCount()
+ {
+ return statusHistoryCount;
+ }
+
+ public void setStatusHistoryCount(int statusHistoryCount)
+ {
+ this.statusHistoryCount = statusHistoryCount;
+ }
+
+ public Queue<String> getMapStatusHistory()
+ {
+ return mapStatusHistory;
+ }
+
+ public Queue<String> getReduceStatusHistory()
+ {
+ return reduceStatusHistory;
+ }
+
+ public Queue<String> getPhysicalMemeoryStatusHistory()
+ {
+ return physicalMemoryStatusHistory;
+ }
+
+ public Queue<String> getVirtualMemoryStatusHistory()
+ {
+ return virtualMemoryStatusHistory;
+ }
+
+ public Queue<String> getCpuStatusHistory()
+ {
+ return cpuStatusHistory;
+ }
+
+ public static class TaskObject
+ {
+ /**
+ * This field stores the task information as json
+ */
+ private JSONObject json;
+ /**
+ * This field tells if the object was modified
+ */
+ private boolean modified;
+
+ public TaskObject(JSONObject json)
+ {
+ modified = true;
+ this.json = json;
+ }
+
+ /**
+ * This returns the task information as json
+ *
+ * @return
+ */
+ public JSONObject getJson()
+ {
+ return json;
+ }
+
+ /**
+ * This stores the task information as json
+ *
+ * @param json
+ */
+ public void setJson(JSONObject json)
+ {
+ this.json = json;
+ }
+
+ /**
+ * This returns if the json object has been modified
+ *
+ * @return
+ */
+ public boolean isModified()
+ {
+ return modified;
+ }
+
+ /**
+ * This sets if the json object is modified
+ *
+ * @param modified
+ */
+ public void setModified(boolean modified)
+ {
+ this.modified = modified;
+ }
+
+ /**
+ * This returns the string format of the json object
+ *
+ * @return
+ */
+ public String getJsonString()
+ {
+ return json.toString();
+ }
+ }
+
+ private static Logger logger = LoggerFactory.getLogger(MRStatusObject.class);
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/mrmonitor/src/main/java/org/apache/apex/examples/mrmonitor/MRUtil.java
----------------------------------------------------------------------
diff --git a/examples/mrmonitor/src/main/java/org/apache/apex/examples/mrmonitor/MRUtil.java b/examples/mrmonitor/src/main/java/org/apache/apex/examples/mrmonitor/MRUtil.java
new file mode 100644
index 0000000..1a82b99
--- /dev/null
+++ b/examples/mrmonitor/src/main/java/org/apache/apex/examples/mrmonitor/MRUtil.java
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.mrmonitor;
+
+import java.io.IOException;
+
+import org.codehaus.jettison.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.http.client.ClientProtocolException;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.ResponseHandler;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.BasicResponseHandler;
+import org.apache.http.impl.client.DefaultHttpClient;
+
+/**
+ * <p>
+ * Util class.
+ * </p>
+ *
+ * @since 0.3.4
+ */
+public class MRUtil
+{
+
+ private static final Logger logger = LoggerFactory.getLogger(MRUtil.class);
+
+ /**
+ * This method returns the response content for a given url
+ * @param url
+ * @return
+ */
+ public static String getJsonForURL(String url)
+ {
+ HttpClient httpclient = new DefaultHttpClient();
+ logger.debug(url);
+ try {
+
+
+ HttpGet httpget = new HttpGet(url);
+
+ // Create a response handler
+ ResponseHandler<String> responseHandler = new BasicResponseHandler();
+ String responseBody;
+ try {
+ responseBody = httpclient.execute(httpget, responseHandler);
+
+ } catch (ClientProtocolException e) {
+ logger.debug(e.getMessage());
+ return null;
+
+ } catch (IOException e) {
+ logger.debug(e.getMessage());
+ return null;
+ } catch (Exception e) {
+ logger.debug(e.getMessage());
+ return null;
+ }
+ return responseBody.trim();
+ } finally {
+ httpclient.getConnectionManager().shutdown();
+ }
+ }
+
+ /**
+ * This method returns the JSONObject for a given string
+ * @param json
+ * @return
+ */
+ public static JSONObject getJsonObject(String json)
+ {
+ try {
+ JSONObject jsonObj = new JSONObject(json);
+ return jsonObj;
+ } catch (Exception e) {
+ logger.debug("{}", e.getMessage());
+ return null;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/mrmonitor/src/main/java/org/apache/apex/examples/mrmonitor/MapToMRObjectOperator.java
----------------------------------------------------------------------
diff --git a/examples/mrmonitor/src/main/java/org/apache/apex/examples/mrmonitor/MapToMRObjectOperator.java b/examples/mrmonitor/src/main/java/org/apache/apex/examples/mrmonitor/MapToMRObjectOperator.java
new file mode 100644
index 0000000..f4ce4fb
--- /dev/null
+++ b/examples/mrmonitor/src/main/java/org/apache/apex/examples/mrmonitor/MapToMRObjectOperator.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.mrmonitor;
+
+import java.util.Map;
+
+import com.datatorrent.api.Context.OperatorContext;
+import com.datatorrent.api.DefaultInputPort;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.Operator;
+
+/**
+ * <p>MapToMRObjectOperator class.</p>
+ *
+ * @since 0.9.0
+ */
+public class MapToMRObjectOperator implements Operator
+{
+
+ public final transient DefaultInputPort<Map<String, String>> input = new DefaultInputPort<Map<String, String>>()
+ {
+ @Override
+ public void process(Map<String, String> tuple)
+ {
+ MRStatusObject mrStatusObj = new MRStatusObject();
+
+ for (Map.Entry<String, String> e : tuple.entrySet()) {
+ if (e.getKey().equals(Constants.QUERY_KEY_COMMAND)) {
+ mrStatusObj.setCommand(e.getValue());
+ } else if (e.getKey().equals(Constants.QUERY_API_VERSION)) {
+ mrStatusObj.setApiVersion(e.getValue());
+ } else if (e.getKey().equals(Constants.QUERY_APP_ID)) {
+ mrStatusObj.setAppId(e.getValue());
+ } else if (e.getKey().equals(Constants.QUERY_HADOOP_VERSION)) {
+ mrStatusObj.setHadoopVersion(Integer.parseInt(e.getValue()));
+ } else if (e.getKey().equals(Constants.QUERY_HOST_NAME)) {
+ mrStatusObj.setUri(e.getValue());
+ } else if (e.getKey().equals(Constants.QUERY_HS_PORT)) {
+ mrStatusObj.setHistoryServerPort(Integer.parseInt(e.getValue()));
+ } else if (e.getKey().equals(Constants.QUERY_JOB_ID)) {
+ mrStatusObj.setJobId(e.getValue());
+ } else if (e.getKey().equals(Constants.QUERY_RM_PORT)) {
+ mrStatusObj.setRmPort(Integer.parseInt(e.getValue()));
+ }
+ }
+ output.emit(mrStatusObj);
+
+ }
+ };
+
+ public final transient DefaultOutputPort<MRStatusObject> output = new DefaultOutputPort<MRStatusObject>();
+
+ @Override
+ public void setup(OperatorContext context)
+ {
+ }
+
+ @Override
+ public void teardown()
+ {
+ }
+
+ @Override
+ public void beginWindow(long windowId)
+ {
+ }
+
+ @Override
+ public void endWindow()
+ {
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/mrmonitor/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/examples/mrmonitor/src/main/resources/META-INF/properties.xml b/examples/mrmonitor/src/main/resources/META-INF/properties.xml
new file mode 100644
index 0000000..fdda52b
--- /dev/null
+++ b/examples/mrmonitor/src/main/resources/META-INF/properties.xml
@@ -0,0 +1,63 @@
+<?xml version="1.0"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<configuration>
+ <property>
+ <name>dt.application.MRMonitoringExample.operator.JobMonitor.attr.PARTITIONER</name>
+ <value>com.datatorrent.common.partitioner.StatelessPartitioner:1</value>
+ </property>
+ <property>
+ <name>dt.application.MRMonitoringExample.operator.JobMonitor.attr.APPLICATION_WINDOW_COUNT</name>
+ <value>4</value>
+ </property>
+ <property>
+ <name>dt.application.MRMonitoringExample.operator.JobMonitor.prop.maxJobs</name>
+ <value>25</value>
+ </property>
+ <property>
+ <name>dt.application.MRMonitoringExample.operator.JobMonitor.prop.maxJobs</name>
+ <value>25</value>
+ </property>
+ <property>
+ <name>dt.application.MRMonitoringExample.operator.Query.prop.topic</name>
+ <value>contrib.summit.mrDebugger.mrDebuggerQuery</value>
+ </property>
+ <property>
+ <name>dt.application.MRMonitoringExample.operator.JobOutput.prop.topic</name>
+ <value>contrib.summit.mrDebugger.jobResult</value>
+ </property>
+ <property>
+ <name>dt.application.MRMonitoringExample.operator.MapJob.prop.topic</name>
+ <value>contrib.summit.mrDebugger.mapResult</value>
+ </property>
+ <property>
+ <name>dt.application.MRMonitoringExample.operator.ReduceJob.prop.topic</name>
+ <value>contrib.summit.mrDebugger.reduceResult</value>
+ </property>
+ <property>
+ <name>dt.application.MRMonitoringExample.operator.JobCounter.prop.topic</name>
+ <value>contrib.summit.mrDebugger.counterResult</value>
+ </property>
+ <property>
+ <name>dt.application.MRMonitoringExample.stream.QueryConversion.locality</name>
+ <value>CONTAINER_LOCAL</value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/mrmonitor/src/main/resources/mrdebugger.html
----------------------------------------------------------------------
diff --git a/examples/mrmonitor/src/main/resources/mrdebugger.html b/examples/mrmonitor/src/main/resources/mrdebugger.html
new file mode 100644
index 0000000..dddde06
--- /dev/null
+++ b/examples/mrmonitor/src/main/resources/mrdebugger.html
@@ -0,0 +1,237 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<!doctype html>
+<html>
+<head>
+<title>Mobile Example</title>
+
+<META HTTP-EQUIV="CACHE-CONTROL" CONTENT="NO-CACHE">
+<meta name="viewport" content="initial-scale=1.0, user-scalable=no">
+<meta charset="utf-8">
+<script src="http://ajax.googleapis.com/ajax/libs/jquery/1.7.1/jquery.min.js"></script>
+<script src="http://ajax.googleapis.com/ajax/libs/jqueryui/1.10.3/jquery-ui.min.js"></script>
+<!--script src="js/vendor/jquery/dist/jquery.js"></script-->
+<style>
+ body {
+ margin: 0;
+ }
+
+ .phone-input {
+ margin-left: 0.5em;
+ margin-right: 0.5em;
+ }
+</style>
+
+</head>
+
+
+<body>
+
+
+<script>
+
+
+
+var map;
+var markers = {};
+
+$(function() {
+
+ $("#query1AddButton").click(function() {
+
+ var app_id = $("input#app_id").val();
+ var hostname = $("input#hostname").val();
+ var job_id = $("input#job_id").val();
+ var hadoop_version= $("input#hadoop_version").val();
+ var api_version = $("input#api_version").val();
+ var rm_port = $("input#rm_port").val();
+ var hs_port = $("input#hs_port").val();
+
+ var jsonData = {
+ command : 'add',
+ hostname:hostname,
+ app_id:app_id,
+ job_id:job_id,
+ hadoop_version:hadoop_version,
+ api_version:api_version,
+ rm_port:rm_port,
+ hs_port:hs_port
+ };
+
+ sendQuery(jsonData, function() {
+ $('#query1SubmitConfirm').html("<div id='message'></div>");
+ $('#message').html("<h2>Add submitted to application!</h2>")
+ .append("<p>Result will appear on page shortly.</p>");
+ });
+
+ return false;
+ });
+
+ $("#query1DeleteButton").click(function() {
+
+ var job_id = $("input#job_id").val();
+
+ var jsonData = {
+ command : 'delete',
+ query : job_id
+ };
+
+ sendQuery(jsonData, function() {
+ $('#query1SubmitConfirm').html("<div id='message'></div>");
+ $('#message').html("<h2>Add " + phone + " submitted to application!</h2>")
+ .append("<p>Result will appear on page shortly.</p>");
+ });
+
+ return false;
+ });
+
+ function sendQuery(jsonData, callback) {
+ var ws = new WebSocket('ws://'+window.location.host+'/pubsub');
+
+ ws.onopen = function () {
+ var topic = "contrib.summit.mrDebugger.mrDebuggerQuery";
+ var msg = JSON.stringify({ "type" : "publish", "topic" : topic, "data" : jsonData });
+ ws.send(msg);
+ console.log("published to: " + topic + " data: " + msg);
+ ws.close();
+ if (callback) callback();
+ };
+
+ ws.onerror = function (error) {
+ console.log('WebSocket Error ' + error);
+ };
+
+ ws.onmessage = function (e) {
+ console.log('Server: ' + e.data);
+ };
+ ws.onclose = function (e) {
+ console.log('close: ' , e);
+ };
+
+ }
+
+ var ws = new WebSocket('ws://'+window.location.host+'/pubsub');
+ var topic = "contrib.summit.mrDebugger.jobResult";
+
+ ws.onopen = function () {
+ var msg = JSON.stringify({ "type":"subscribe", "topic": topic});
+ console.log("sending: " + msg);
+ ws.send(msg);
+ };
+
+ ws.onerror = function (error) {
+ console.log('WebSocket Error ' + error);
+ };
+
+ ws.onmessage = function (e){
+
+ $('#jobQueryResult').append(e.data+"\n");
+ };
+
+
+ var mapws = new WebSocket('ws://'+window.location.host+'/pubsub');
+ var maptopic = "contrib.summit.mrDebugger.mapResult";
+
+ mapws.onopen = function () {
+ var msg = JSON.stringify({ "type":"subscribe", "topic": maptopic});
+ console.log("sending: " + msg);
+ mapws.send(msg);
+ };
+
+ mapws.onerror = function (error) {
+ console.log('WebSocket Error ' + error);
+ };
+
+ mapws.onmessage = function (e){
+
+ $('#jobMapQueryResult').append(e.data+"\n");
+ };
+
+
+ var reducews = new WebSocket('ws://'+window.location.host+'/pubsub');
+ var reducetopic = "contrib.summit.mrDebugger.reduceResult";
+
+ reducews.onopen = function () {
+ var msg = JSON.stringify({ "type":"subscribe", "topic": reducetopic});
+ console.log("sending: " + msg);
+ reducews.send(msg);
+ };
+
+ reducews.onerror = function (error) {
+ console.log('WebSocket Error ' + error);
+ };
+
+ reducews.onmessage = function (e){
+
+ $('#jobReduceQueryResult').append(e.data+"\n");
+ };
+
+ });
+
+
+</script>
+
+
+<div id="query1FormDiv">
+ <form name="query1" action="">
+ <p>
+ <label for="phone" id="app_id_label">Application Id</label>
+ <input type="text" name="app_id" id="app_id" size="30" value="" class="phone-input" />
+ </p>
+ <p>
+ <label for="phone" id="job_id_label">Job Id</label>
+ <input type="text" name="job_id" id="job_id" size="30" value="" class="phone-input" />
+ </p>
+ <p>
+ <label for="phone" id="hostname_label">Hostname</label>
+ <input type="text" name="hostname" id="hostname" size="30" value="" class="phone-input" />
+ </p>
+ <p>
+ <label for="phone" id="rm_port_label">RM port</label>
+ <input type="text" name="rm_port" id="rm_port" size="30" value="" class="phone-input" />
+ </p>
+ <p>
+ <label for="phone" id="hs_port_label">History Server port</label>
+ <input type="text" name="hs_port" id="hs_port" size="30" value="" class="phone-input" />
+ </p>
+ <p>
+ <label for="phone" id="hadoop_version_label">Hadoop Version</label>
+ <input type="text" name="hadoop_version" id="hadoop_version" size="30" value="" class="phone-input" />
+ </p>
+ <p>
+ <label for="phone" id="api_version_label">API Version</label>
+ <input type="text" name="api_version" id="api_version" size="30" value="" class="phone-input" />
+ </p>
+ <p>
+ <input type="submit" name="command" class="button" id="query1AddButton" value="Add" />
+ <input type="submit" name="command" class="button" id="query1DeleteButton" value="Delete" />
+ <input type="submit" name="command" class="button" id="query1ClearButton" value="Clear" />
+ </p>
+ </form>
+ <div id="query1SubmitConfirm"></div>
+ <div>Job: <span id="jobQueryResult"></span></div>
+ <div>Map Task: <span id="jobMapQueryResult"></span></div>
+ <div>Reduce Job: <span id="jobReduceQueryResult"></span></div>
+ </div>
+
+</body>
+</html>
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/mrmonitor/src/test/java/org/apache/apex/examples/mrmonitor/MrMonitoringApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/mrmonitor/src/test/java/org/apache/apex/examples/mrmonitor/MrMonitoringApplicationTest.java b/examples/mrmonitor/src/test/java/org/apache/apex/examples/mrmonitor/MrMonitoringApplicationTest.java
new file mode 100644
index 0000000..b094ddb
--- /dev/null
+++ b/examples/mrmonitor/src/test/java/org/apache/apex/examples/mrmonitor/MrMonitoringApplicationTest.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.mrmonitor;
+
+import javax.servlet.Servlet;
+
+import org.eclipse.jetty.server.Connector;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.servlet.ServletContextHandler;
+import org.eclipse.jetty.servlet.ServletHolder;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.LocalMode;
+import com.datatorrent.lib.helper.SamplePubSubWebSocketServlet;
+
+/**
+ * <p>MapReduceDebuggerApplicationTest class.</p>
+ *
+ * @since 0.3.4
+ */
+
+public class MrMonitoringApplicationTest
+{
+
+ @Test
+ public void testApplication() throws Exception
+ {
+ Configuration conf = new Configuration(false);
+ conf.addResource("dt-site-monitoring.xml");
+ Server server = new Server(0);
+ Servlet servlet = new SamplePubSubWebSocketServlet();
+ ServletHolder sh = new ServletHolder(servlet);
+ ServletContextHandler contextHandler = new ServletContextHandler(server, "/", ServletContextHandler.SESSIONS);
+ contextHandler.addServlet(sh, "/pubsub");
+ contextHandler.addServlet(sh, "/*");
+ server.start();
+ Connector[] connector = server.getConnectors();
+ conf.set("dt.attr.GATEWAY_CONNECT_ADDRESS", "localhost:" + connector[0].getLocalPort());
+
+ MRMonitoringApplication application = new MRMonitoringApplication();
+ LocalMode lma = LocalMode.newInstance();
+ lma.prepareDAG(application, conf);
+ LocalMode.Controller lc = lma.getController();
+ lc.run(10000);
+ server.stop();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/mrmonitor/src/test/resources/dt-site-monitoring.xml
----------------------------------------------------------------------
diff --git a/examples/mrmonitor/src/test/resources/dt-site-monitoring.xml b/examples/mrmonitor/src/test/resources/dt-site-monitoring.xml
new file mode 100644
index 0000000..88bae44
--- /dev/null
+++ b/examples/mrmonitor/src/test/resources/dt-site-monitoring.xml
@@ -0,0 +1,63 @@
+<?xml version="1.0"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<configuration>
+ <property>
+ <name>dt.application.MRMonitoringExample.operator.JobMonitor.attr.INITIAL_PARTITION_COUNT</name>
+ <value>1</value>
+ </property>
+ <property>
+ <name>dt.application.MRMonitoringExample.operator.JobMonitor.attr.APPLICATION_WINDOW_COUNT</name>
+ <value>4</value>
+ </property>
+ <property>
+ <name>dt.application.MRMonitoringExample.operator.JobMonitor.prop.maxJobs</name>
+ <value>25</value>
+ </property>
+ <property>
+ <name>dt.application.MRMonitoringExample.operator.JobMonitor.prop.maxJobs</name>
+ <value>25</value>
+ </property>
+ <property>
+ <name>dt.application.MRMonitoringExample.operator.Query.prop.topic</name>
+ <value>contrib.summit.mrDebugger.mrDebuggerQuery</value>
+ </property>
+ <property>
+ <name>dt.application.MRMonitoringExample.operator.JobOutput.prop.topic</name>
+ <value>contrib.summit.mrDebugger.jobResult</value>
+ </property>
+ <property>
+ <name>dt.application.MRMonitoringExample.operator.MapJob.prop.topic</name>
+ <value>contrib.summit.mrDebugger.mapResult</value>
+ </property>
+ <property>
+ <name>dt.application.MRMonitoringExample.operator.ReduceJob.prop.topic</name>
+ <value>contrib.summit.mrDebugger.reduceResult</value>
+ </property>
+ <property>
+ <name>dt.application.MRMonitoringExample.operator.JobCounter.prop.topic</name>
+ <value>contrib.summit.mrDebugger.counterResult</value>
+ </property>
+ <property>
+ <name>dt.application.MRMonitoringExample.stream.QueryConversion.locality</name>
+ <value>CONTAINER_LOCAL</value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/mrmonitor/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/examples/mrmonitor/src/test/resources/log4j.properties b/examples/mrmonitor/src/test/resources/log4j.properties
new file mode 100644
index 0000000..cf0d19e
--- /dev/null
+++ b/examples/mrmonitor/src/test/resources/log4j.properties
@@ -0,0 +1,43 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+log4j.rootLogger=DEBUG,CONSOLE
+
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+log4j.appender.CONSOLE.threshold=${test.log.console.threshold}
+test.log.console.threshold=DEBUG
+
+log4j.appender.RFA=org.apache.log4j.RollingFileAppender
+log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
+log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+log4j.appender.RFA.File=/tmp/app.log
+
+# to enable, add SYSLOG to rootLogger
+log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender
+log4j.appender.SYSLOG.syslogHost=127.0.0.1
+log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout
+log4j.appender.SYSLOG.layout.conversionPattern=${dt.cid} %-5p [%t] %c{2} %x - %m%n
+log4j.appender.SYSLOG.Facility=LOCAL1
+
+log4j.logger.org=info
+#log4j.logger.org.apache.commons.beanutils=warn
+log4j.logger.com.datatorrent=debug
+log4j.logger.org.apache.apex=debug
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/mroperator/pom.xml
----------------------------------------------------------------------
diff --git a/examples/mroperator/pom.xml b/examples/mroperator/pom.xml
new file mode 100644
index 0000000..c9a7b65
--- /dev/null
+++ b/examples/mroperator/pom.xml
@@ -0,0 +1,56 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>malhar-examples-mroperator</artifactId>
+ <packaging>jar</packaging>
+
+ <name>Apache Apex Malhar MR Operator Example</name>
+ <description></description>
+
+ <parent>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-examples</artifactId>
+ <version>3.7.0-SNAPSHOT</version>
+ </parent>
+
+ <properties>
+ <skipTests>true</skipTests>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/mroperator/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/examples/mroperator/src/assemble/appPackage.xml b/examples/mroperator/src/assemble/appPackage.xml
new file mode 100644
index 0000000..4138cf2
--- /dev/null
+++ b/examples/mroperator/src/assemble/appPackage.xml
@@ -0,0 +1,59 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+ <id>appPackage</id>
+ <formats>
+ <format>jar</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>${basedir}/target/</directory>
+ <outputDirectory>/app</outputDirectory>
+ <includes>
+ <include>${project.artifactId}-${project.version}.jar</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/target/deps</directory>
+ <outputDirectory>/lib</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/site/conf</directory>
+ <outputDirectory>/conf</outputDirectory>
+ <includes>
+ <include>*.xml</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/META-INF</directory>
+ <outputDirectory>/META-INF</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/app</directory>
+ <outputDirectory>/app</outputDirectory>
+ </fileSet>
+ </fileSets>
+
+</assembly>
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/DateWritable.java
----------------------------------------------------------------------
diff --git a/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/DateWritable.java b/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/DateWritable.java
new file mode 100644
index 0000000..4cf1f5a
--- /dev/null
+++ b/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/DateWritable.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.mroperator;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+import org.apache.hadoop.io.WritableComparable;
+
+/**
+ * <p>DateWritable class.</p>
+ *
+ * @since 0.9.0
+ */
+public class DateWritable implements WritableComparable<DateWritable>
+{
+ private static final SimpleDateFormat formatter = new SimpleDateFormat( "yyyy-MM-dd' T 'HH:mm:ss.SSS" );
+ private Date date;
+
+ public Date getDate()
+ {
+ return date;
+ }
+
+ public void setDate( Date date )
+ {
+ this.date = date;
+ }
+
+ public void readFields( DataInput in ) throws IOException
+ {
+ date = new Date( in.readLong() );
+ }
+
+ public void write( DataOutput out ) throws IOException
+ {
+ out.writeLong( date.getTime() );
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ return toString().equals(o.toString());
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return toString().hashCode();
+ }
+
+ public String toString()
+ {
+ return formatter.format( date);
+ }
+
+ public int compareTo( DateWritable other )
+ {
+ return date.compareTo( other.getDate() );
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/HdfsKeyValOutputOperator.java
----------------------------------------------------------------------
diff --git a/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/HdfsKeyValOutputOperator.java b/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/HdfsKeyValOutputOperator.java
new file mode 100644
index 0000000..94e17c1
--- /dev/null
+++ b/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/HdfsKeyValOutputOperator.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.mroperator;
+
+import com.datatorrent.lib.io.fs.AbstractSingleFileOutputOperator;
+import com.datatorrent.lib.util.KeyHashValPair;
+
+/**
+ * Adapter for writing KeyHashValPair objects to HDFS
+ * <p>
+ * Serializes tuples into a HDFS file.<br/>
+ * </p>
+ *
+ * @param <K> Key type
+ * @param <V> Value type
+ * @since 0.9.4
+ */
+public class HdfsKeyValOutputOperator<K, V> extends AbstractSingleFileOutputOperator<KeyHashValPair<K, V>>
+{
+ @Override
+ public byte[] getBytesForTuple(KeyHashValPair<K,V> t)
+ {
+ return (t.toString() + "\n").getBytes();
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/InvertedIndexApplication.java
----------------------------------------------------------------------
diff --git a/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/InvertedIndexApplication.java b/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/InvertedIndexApplication.java
new file mode 100644
index 0000000..af97bc4
--- /dev/null
+++ b/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/InvertedIndexApplication.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.mroperator;
+
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.TextInputFormat;
+
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+
+/**
+ * <p>InvertedIndexApplication class.</p>
+ *
+ * @since 0.9.0
+ */
+@ApplicationAnnotation(name = "InvertedIndexExample")
+public class InvertedIndexApplication extends MapReduceApplication<LongWritable, Text, Text, Text>
+{
+
+ InvertedIndexApplication()
+ {
+ setMapClass(LineIndexer.LineIndexMapper.class);
+ setReduceClass(LineIndexer.LineIndexReducer.class);
+
+ setInputFormat(TextInputFormat.class);
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/LineIndexer.java
----------------------------------------------------------------------
diff --git a/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/LineIndexer.java b/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/LineIndexer.java
new file mode 100644
index 0000000..a2c589d
--- /dev/null
+++ b/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/LineIndexer.java
@@ -0,0 +1,120 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.mroperator;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.StringTokenizer;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.FileOutputFormat;
+import org.apache.hadoop.mapred.FileSplit;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.MapReduceBase;
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reducer;
+import org.apache.hadoop.mapred.Reporter;
+
+/**
+ * <p>LineIndexer class.</p>
+ *
+ * @since 0.9.0
+ */
+public class LineIndexer
+{
+
+ public static class LineIndexMapper extends MapReduceBase
+ implements Mapper<LongWritable, Text, Text, Text>
+ {
+ private static final Text word = new Text();
+ private static final Text location = new Text();
+
+ public void map(LongWritable key, Text val,
+ OutputCollector<Text, Text> output, Reporter reporter) throws IOException
+ {
+ FileSplit fileSplit = (FileSplit)reporter.getInputSplit();
+ String fileName = fileSplit.getPath().getName();
+ location.set(fileName);
+
+ String line = val.toString();
+ StringTokenizer itr = new StringTokenizer(line.toLowerCase());
+ while (itr.hasMoreTokens()) {
+ word.set(itr.nextToken());
+ output.collect(word, location);
+ }
+ }
+ }
+
+
+
+ public static class LineIndexReducer extends MapReduceBase
+ implements Reducer<Text, Text, Text, Text>
+ {
+ public void reduce(Text key, Iterator<Text> values,
+ OutputCollector<Text, Text> output, Reporter reporter) throws IOException
+ {
+ boolean first = true;
+ StringBuilder toReturn = new StringBuilder();
+ while (values.hasNext()) {
+ if (!first) {
+ toReturn.append(", ");
+ }
+ first = false;
+ toReturn.append(values.next().toString());
+ }
+
+ output.collect(key, new Text(toReturn.toString()));
+ }
+ }
+
+
+ /**
+ * The actual main() method for our program; this is the
+ * "driver" for the MapReduce job.
+ */
+ public static void main(String[] args)
+ {
+ JobClient client = new JobClient();
+ JobConf conf = new JobConf(LineIndexer.class);
+
+ conf.setJobName("LineIndexer");
+
+ conf.setOutputKeyClass(Text.class);
+ conf.setOutputValueClass(Text.class);
+
+ FileInputFormat.addInputPath(conf, new Path("input"));
+ FileOutputFormat.setOutputPath(conf, new Path("output"));
+
+ conf.setMapperClass(LineIndexMapper.class);
+ conf.setReducerClass(LineIndexReducer.class);
+
+ client.setConf(conf);
+
+ try {
+ JobClient.runJob(conf);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/LogCountsPerHour.java
----------------------------------------------------------------------
diff --git a/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/LogCountsPerHour.java b/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/LogCountsPerHour.java
new file mode 100644
index 0000000..6255810
--- /dev/null
+++ b/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/LogCountsPerHour.java
@@ -0,0 +1,187 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.mroperator;
+
+import java.io.IOException;
+import java.util.Calendar;
+import java.util.Iterator;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.FileOutputFormat;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.MapReduceBase;
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reducer;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.TextOutputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+/**
+ * <p>LogCountsPerHour class.</p>
+ *
+ * @since 0.9.0
+ */
+public class LogCountsPerHour extends Configured implements Tool
+{
+
+ public static class LogMapClass extends MapReduceBase
+ implements Mapper<LongWritable, Text, DateWritable, IntWritable>
+ {
+ private DateWritable date = new DateWritable();
+ private static final IntWritable one = new IntWritable(1);
+
+ public void map(LongWritable key, Text value, OutputCollector<DateWritable, IntWritable> output, Reporter reporter) throws IOException
+ {
+ // Get the value as a String; it is of the format:
+ // 111.111.111.111 - - [16/Dec/2012:05:32:50 -0500] "GET / HTTP/1.1" 200 14791 "-" "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)"
+ String text = value.toString();
+
+ // Get the date and time
+ int openBracket = text.indexOf('[');
+ int closeBracket = text.indexOf(']');
+ if (openBracket != -1 && closeBracket != -1) {
+ // Read the date
+ String dateString = text.substring(text.indexOf('[') + 1, text.indexOf(']'));
+
+ // Build a date object from a string of the form: 16/Dec/2012:05:32:50 -0500
+ int index = 0;
+ int nextIndex = dateString.indexOf('/');
+ int day = Integer.parseInt(dateString.substring(index, nextIndex));
+
+ index = nextIndex;
+ nextIndex = dateString.indexOf('/', index + 1);
+ String month = dateString.substring(index + 1, nextIndex);
+
+ index = nextIndex;
+ nextIndex = dateString.indexOf(':', index);
+ int year = Integer.parseInt(dateString.substring(index + 1, nextIndex));
+
+ index = nextIndex;
+ nextIndex = dateString.indexOf(':', index + 1);
+ int hour = Integer.parseInt(dateString.substring(index + 1, nextIndex));
+
+ // Build a calendar object for this date
+ Calendar calendar = Calendar.getInstance();
+ calendar.set(Calendar.DATE, day);
+ calendar.set(Calendar.YEAR, year);
+ calendar.set(Calendar.HOUR, hour);
+ calendar.set(Calendar.MINUTE, 0);
+ calendar.set(Calendar.SECOND, 0);
+ calendar.set(Calendar.MILLISECOND, 0);
+
+ if (month.equalsIgnoreCase("dec")) {
+ calendar.set(Calendar.MONTH, Calendar.DECEMBER);
+ } else if (month.equalsIgnoreCase("nov")) {
+ calendar.set(Calendar.MONTH, Calendar.NOVEMBER);
+ } else if (month.equalsIgnoreCase("oct")) {
+ calendar.set(Calendar.MONTH, Calendar.OCTOBER);
+ } else if (month.equalsIgnoreCase("sep")) {
+ calendar.set(Calendar.MONTH, Calendar.SEPTEMBER);
+ } else if (month.equalsIgnoreCase("aug")) {
+ calendar.set(Calendar.MONTH, Calendar.AUGUST);
+ } else if (month.equalsIgnoreCase("jul")) {
+ calendar.set(Calendar.MONTH, Calendar.JULY);
+ } else if (month.equalsIgnoreCase("jun")) {
+ calendar.set(Calendar.MONTH, Calendar.JUNE);
+ } else if (month.equalsIgnoreCase("may")) {
+ calendar.set(Calendar.MONTH, Calendar.MAY);
+ } else if (month.equalsIgnoreCase("apr")) {
+ calendar.set(Calendar.MONTH, Calendar.APRIL);
+ } else if (month.equalsIgnoreCase("mar")) {
+ calendar.set(Calendar.MONTH, Calendar.MARCH);
+ } else if (month.equalsIgnoreCase("feb")) {
+ calendar.set(Calendar.MONTH, Calendar.FEBRUARY);
+ } else if (month.equalsIgnoreCase("jan")) {
+ calendar.set(Calendar.MONTH, Calendar.JANUARY);
+ }
+
+
+ // Output the date as the key and 1 as the value
+ date.setDate(calendar.getTime());
+ output.collect(date, one);
+ }
+ }
+ }
+
+ public static class LogReduce extends MapReduceBase
+ implements Reducer<DateWritable, IntWritable, DateWritable, IntWritable>
+ {
+ public void reduce(DateWritable key, Iterator<IntWritable> values, OutputCollector<DateWritable, IntWritable> output, Reporter reporter) throws IOException
+ {
+ // Iterate over all of the values (counts of occurrences of this word)
+ int count = 0;
+ while (values.hasNext()) {
+ // Add the value to our count
+ count += values.next().get();
+ }
+
+ // Output the word with its count (wrapped in an IntWritable)
+ output.collect(key, new IntWritable(count));
+ }
+ }
+
+
+ public int run(String[] args) throws Exception
+ {
+ // Create a configuration
+ Configuration conf = getConf();
+
+ // Create a job from the default configuration that will use the WordCount class
+ JobConf job = new JobConf(conf, LogCountsPerHour.class);
+
+ // Define our input path as the first command line argument and our output path as the second
+ Path in = new Path(args[0]);
+ Path out = new Path(args[1]);
+
+ // Create File Input/Output formats for these paths (in the job)
+ FileInputFormat.setInputPaths(job, in);
+ FileOutputFormat.setOutputPath(job, out);
+
+ // Configure the job: name, mapper, reducer, and combiner
+ job.setJobName("LogAveragePerHour");
+ job.setMapperClass(LogMapClass.class);
+ job.setReducerClass(LogReduce.class);
+ job.setCombinerClass(LogReduce.class);
+
+ // Configure the output
+ job.setOutputFormat(TextOutputFormat.class);
+ job.setOutputKeyClass(DateWritable.class);
+ job.setOutputValueClass(IntWritable.class);
+
+ // Run the job
+ JobClient.runJob(job);
+ return 0;
+ }
+
+ public static void main(String[] args) throws Exception
+ {
+ // Start the LogCountsPerHour MapReduce application
+ int res = ToolRunner.run(new Configuration(), new LogCountsPerHour(), args);
+ System.exit(res);
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/LogsCountApplication.java
----------------------------------------------------------------------
diff --git a/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/LogsCountApplication.java b/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/LogsCountApplication.java
new file mode 100644
index 0000000..51b082d
--- /dev/null
+++ b/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/LogsCountApplication.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.mroperator;
+
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.TextInputFormat;
+
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+
+/**
+ * <p>LogsCountApplication class.</p>
+ *
+ * @since 0.9.0
+ */
+@ApplicationAnnotation(name = "LogsCountExample")
+public class LogsCountApplication extends MapReduceApplication<LongWritable, Text, DateWritable, IntWritable>
+{
+
+ public void LogsCountApplication()
+ {
+ setMapClass(LogCountsPerHour.LogMapClass.class);
+ // setCombineClass(LogCountsPerHour.LogReduce.class);
+ setReduceClass(LogCountsPerHour.LogReduce.class);
+ setInputFormat(TextInputFormat.class);
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/MapOperator.java
----------------------------------------------------------------------
diff --git a/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/MapOperator.java b/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/MapOperator.java
new file mode 100644
index 0000000..ce00f54
--- /dev/null
+++ b/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/MapOperator.java
@@ -0,0 +1,414 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.mroperator;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import javax.validation.constraints.Min;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.apex.examples.mroperator.ReporterImpl.ReporterType;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.serializer.Deserializer;
+import org.apache.hadoop.io.serializer.SerializationFactory;
+import org.apache.hadoop.io.serializer.Serializer;
+import org.apache.hadoop.mapred.Counters;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.mapred.InputSplit;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.KeyValueTextInputFormat;
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.RecordReader;
+import org.apache.hadoop.mapred.Reducer;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.TextInputFormat;
+
+import com.datatorrent.api.Context.OperatorContext;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.DefaultPartition;
+import com.datatorrent.api.InputOperator;
+import com.datatorrent.api.Partitioner;
+
+import com.datatorrent.lib.util.KeyHashValPair;
+
+/**
+ * <p>
+ * MapOperator class.
+ * </p>
+ *
+ * @since 0.9.0
+ */
+@SuppressWarnings({ "unchecked"})
+public class MapOperator<K1, V1, K2, V2> implements InputOperator, Partitioner<MapOperator<K1, V1, K2, V2>>
+{
+
+ private static final Logger logger = LoggerFactory.getLogger(MapOperator.class);
+ private String dirName;
+ private boolean emitPartitioningCountOnce = false;
+ private boolean emitLastCountOnce = false;
+ private int operatorId;
+ private Class<? extends InputFormat<K1, V1>> inputFormatClass;
+ private transient InputFormat<K1, V1> inputFormat;
+ private transient InputSplit inputSplit;
+ private Class<? extends InputSplit> inputSplitClass;
+ private ByteArrayOutputStream outstream = new ByteArrayOutputStream();
+ private transient RecordReader<K1, V1> reader;
+ private boolean emittedAll = false;
+ public final transient DefaultOutputPort<KeyHashValPair<Integer, Integer>> outputCount = new DefaultOutputPort<KeyHashValPair<Integer, Integer>>();
+ public final transient DefaultOutputPort<KeyHashValPair<K2, V2>> output = new DefaultOutputPort<KeyHashValPair<K2, V2>>();
+ private transient JobConf jobConf;
+ @Min(1)
+ private int partitionCount = 1;
+
+ public Class<? extends InputSplit> getInputSplitClass()
+ {
+ return inputSplitClass;
+ }
+
+ public void setInputSplitClass(Class<? extends InputSplit> inputSplitClass)
+ {
+ this.inputSplitClass = inputSplitClass;
+ }
+
+ public Class<? extends InputFormat<K1, V1>> getInputFormatClass()
+ {
+ return inputFormatClass;
+ }
+
+ public void setInputFormatClass(Class<? extends InputFormat<K1, V1>> inputFormatClass)
+ {
+ this.inputFormatClass = inputFormatClass;
+ }
+
+ public String getDirName()
+ {
+ return dirName;
+ }
+
+ public void setDirName(String dirName)
+ {
+ this.dirName = dirName;
+ }
+
+ public int getPartitionCount()
+ {
+ return partitionCount;
+ }
+
+ public void setPartitionCount(int partitionCount)
+ {
+ this.partitionCount = partitionCount;
+ }
+
+ @Override
+ public void beginWindow(long windowId)
+ {
+ if (!emitPartitioningCountOnce) {
+ outputCount.emit(new KeyHashValPair<Integer, Integer>(operatorId, 1));
+ emitPartitioningCountOnce = true;
+ }
+ if (reader == null) {
+ try {
+ reader = inputFormat.getRecordReader(inputSplit, new JobConf(new Configuration()), reporter);
+ } catch (IOException e) {
+ logger.info("error getting record reader {}", e.getMessage());
+ }
+ }
+ }
+
+ @Override
+ public void teardown()
+ {
+
+ }
+
+ @Override
+ public void setup(OperatorContext context)
+ {
+ if (context != null) {
+ operatorId = context.getId();
+ }
+ reporter = new ReporterImpl(ReporterType.Mapper, new Counters());
+ outputCollector = new OutputCollectorImpl<K2, V2>();
+ Configuration conf = new Configuration();
+ try {
+ inputFormat = inputFormatClass.newInstance();
+ SerializationFactory serializationFactory = new SerializationFactory(conf);
+ Deserializer keyDesiralizer = serializationFactory.getDeserializer(inputSplitClass);
+ keyDesiralizer.open(new ByteArrayInputStream(outstream.toByteArray()));
+ inputSplit = (InputSplit)keyDesiralizer.deserialize(null);
+ ((ReporterImpl)reporter).setInputSplit(inputSplit);
+ reader = inputFormat.getRecordReader(inputSplit, new JobConf(conf), reporter);
+ } catch (Exception e) {
+ logger.info("failed to initialize inputformat obj {}", inputFormat);
+ throw new RuntimeException(e);
+ }
+ InputStream stream = null;
+ if (configFile != null && configFile.length() > 0) {
+ stream = ClassLoader.getSystemResourceAsStream("/" + configFile);
+ if (stream == null) {
+ stream = ClassLoader.getSystemResourceAsStream(configFile);
+ }
+ }
+ if (stream != null) {
+ conf.addResource(stream);
+ }
+ jobConf = new JobConf(conf);
+ if (mapClass != null) {
+ try {
+ mapObject = mapClass.newInstance();
+ } catch (Exception e) {
+ logger.info("can't instantiate object {}", e.getMessage());
+ }
+
+ mapObject.configure(jobConf);
+ }
+ if (combineClass != null) {
+ try {
+ combineObject = combineClass.newInstance();
+ } catch (Exception e) {
+ logger.info("can't instantiate object {}", e.getMessage());
+ }
+ combineObject.configure(jobConf);
+ }
+ }
+
+ @Override
+ public void emitTuples()
+ {
+ if (!emittedAll) {
+ try {
+ K1 key = reader.createKey();
+ V1 val = reader.createValue();
+ emittedAll = !reader.next(key, val);
+ if (!emittedAll) {
+ KeyHashValPair<K1, V1> keyValue = new KeyHashValPair<K1, V1>(key, val);
+ mapObject.map(keyValue.getKey(), keyValue.getValue(), outputCollector, reporter);
+ if (combineObject == null) {
+ List<KeyHashValPair<K2, V2>> list = ((OutputCollectorImpl<K2, V2>)outputCollector).getList();
+ for (KeyHashValPair<K2, V2> e : list) {
+ output.emit(e);
+ }
+ list.clear();
+ }
+ }
+ } catch (IOException ex) {
+ logger.debug(ex.toString());
+ throw new RuntimeException(ex);
+ }
+ }
+ }
+
+ @Override
+ public void endWindow()
+ {
+ List<KeyHashValPair<K2, V2>> list = ((OutputCollectorImpl<K2, V2>)outputCollector).getList();
+ if (combineObject != null) {
+ Map<K2, List<V2>> cacheObject = new HashMap<K2, List<V2>>();
+ for (KeyHashValPair<K2, V2> tuple : list) {
+ List<V2> cacheList = cacheObject.get(tuple.getKey());
+ if (cacheList == null) {
+ cacheList = new ArrayList<V2>();
+ cacheList.add(tuple.getValue());
+ cacheObject.put(tuple.getKey(), cacheList);
+ } else {
+ cacheList.add(tuple.getValue());
+ }
+ }
+ list.clear();
+ OutputCollector<K2, V2> tempOutputCollector = new OutputCollectorImpl<K2, V2>();
+ for (Map.Entry<K2, List<V2>> e : cacheObject.entrySet()) {
+ try {
+ combineObject.reduce(e.getKey(), e.getValue().iterator(), tempOutputCollector, reporter);
+ } catch (IOException e1) {
+ logger.info(e1.getMessage());
+ }
+ }
+ list = ((OutputCollectorImpl<K2, V2>)tempOutputCollector).getList();
+ for (KeyHashValPair<K2, V2> e : list) {
+ output.emit(e);
+ }
+ }
+ if (!emitLastCountOnce && emittedAll) {
+ outputCount.emit(new KeyHashValPair<Integer, Integer>(operatorId, -1));
+ logger.info("emitting end of file {}", new KeyHashValPair<Integer, Integer>(operatorId, -1));
+ emitLastCountOnce = true;
+ }
+ list.clear();
+ }
+
+ private InputSplit[] getSplits(JobConf conf, int numSplits, String path) throws Exception
+ {
+ FileInputFormat.setInputPaths(conf, new Path(path));
+ if (inputFormat == null) {
+ inputFormat = inputFormatClass.newInstance();
+ String inputFormatClassName = inputFormatClass.getName();
+ if (inputFormatClassName.equals("org.apache.hadoop.mapred.TextInputFormat")) {
+ ((TextInputFormat)inputFormat).configure(conf);
+ } else if (inputFormatClassName.equals("org.apache.hadoop.mapred.KeyValueTextInputFormat")) {
+ ((KeyValueTextInputFormat)inputFormat).configure(conf);
+ }
+ }
+ return inputFormat.getSplits(conf, numSplits);
+ // return null;
+ }
+
+ @Override
+ public void partitioned(Map<Integer, Partition<MapOperator<K1, V1, K2, V2>>> partitions)
+ {
+ }
+
+ @SuppressWarnings("rawtypes")
+ @Override
+ public Collection<Partition<MapOperator<K1, V1, K2, V2>>> definePartitions(Collection<Partition<MapOperator<K1, V1, K2, V2>>> partitions, PartitioningContext context)
+ {
+ int tempPartitionCount = partitionCount;
+
+ Collection c = partitions;
+ Collection<Partition<MapOperator<K1, V1, K2, V2>>> operatorPartitions = c;
+ Partition<MapOperator<K1, V1, K2, V2>> template;
+ Iterator<Partition<MapOperator<K1, V1, K2, V2>>> itr = operatorPartitions.iterator();
+ template = itr.next();
+ Configuration conf = new Configuration();
+ SerializationFactory serializationFactory = new SerializationFactory(conf);
+ if (outstream.size() == 0) {
+ InputSplit[] splits;
+ try {
+ splits = getSplits(new JobConf(conf), tempPartitionCount, template.getPartitionedInstance().getDirName());
+ } catch (Exception e1) {
+ logger.info(" can't get splits {}", e1.getMessage());
+ throw new RuntimeException(e1);
+ }
+ Collection<Partition<MapOperator<K1, V1, K2, V2>>> operList = new ArrayList<Partition<MapOperator<K1, V1, K2, V2>>>();
+ itr = operatorPartitions.iterator();
+ int size = splits.length;
+ Serializer keySerializer = serializationFactory.getSerializer(splits[0].getClass());
+ while (size > 0 && itr.hasNext()) {
+ Partition<MapOperator<K1, V1, K2, V2>> p = itr.next();
+ MapOperator<K1, V1, K2, V2> opr = p.getPartitionedInstance();
+ opr.setInputFormatClass(inputFormatClass);
+ opr.setMapClass(mapClass);
+ opr.setCombineClass(combineClass);
+ opr.setConfigFile(configFile);
+ try {
+ keySerializer.open(opr.getOutstream());
+ keySerializer.serialize(splits[size - 1]);
+ opr.setInputSplitClass(splits[size - 1].getClass());
+ } catch (IOException e) {
+ logger.info("error while serializing {}", e.getMessage());
+ }
+ size--;
+ operList.add(p);
+ }
+ while (size > 0) {
+ MapOperator<K1, V1, K2, V2> opr = new MapOperator<K1, V1, K2, V2>();
+ opr.setInputFormatClass(inputFormatClass);
+ opr.setMapClass(mapClass);
+ opr.setCombineClass(combineClass);
+ opr.setConfigFile(configFile);
+ try {
+ keySerializer.open(opr.getOutstream());
+ keySerializer.serialize(splits[size - 1]);
+ opr.setInputSplitClass(splits[size - 1].getClass());
+ } catch (IOException e) {
+ logger.info("error while serializing {}", e.getMessage());
+ }
+ size--;
+ operList.add(new DefaultPartition<MapOperator<K1, V1, K2, V2>>(opr));
+ }
+ try {
+ keySerializer.close();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return operList;
+ }
+ return null;
+ }
+
+ public ByteArrayOutputStream getOutstream()
+ {
+ return outstream;
+ }
+
+ public void setOutstream(ByteArrayOutputStream outstream)
+ {
+ this.outstream = outstream;
+ }
+
+ /**
+ * adding map code
+ */
+
+ private Class<? extends Mapper<K1, V1, K2, V2>> mapClass;
+ private Class<? extends Reducer<K2, V2, K2, V2>> combineClass;
+
+ private transient Mapper<K1, V1, K2, V2> mapObject;
+ private transient Reducer<K2, V2, K2, V2> combineObject;
+ private transient Reporter reporter;
+
+ private String configFile;
+
+ public String getConfigFile()
+ {
+ return configFile;
+ }
+
+ public void setConfigFile(String configFile)
+ {
+ this.configFile = configFile;
+ }
+
+ private transient OutputCollector<K2, V2> outputCollector;
+
+ public Class<? extends Mapper<K1, V1, K2, V2>> getMapClass()
+ {
+ return mapClass;
+ }
+
+ public void setMapClass(Class<? extends Mapper<K1, V1, K2, V2>> mapClass)
+ {
+ this.mapClass = mapClass;
+ }
+
+ public Class<? extends Reducer<K2, V2, K2, V2>> getCombineClass()
+ {
+ return combineClass;
+ }
+
+ public void setCombineClass(Class<? extends Reducer<K2, V2, K2, V2>> combineClass)
+ {
+ this.combineClass = combineClass;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/MapReduceApplication.java
----------------------------------------------------------------------
diff --git a/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/MapReduceApplication.java b/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/MapReduceApplication.java
new file mode 100644
index 0000000..98f4dc7
--- /dev/null
+++ b/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/MapReduceApplication.java
@@ -0,0 +1,114 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.mroperator;
+
+import java.util.StringTokenizer;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.Reducer;
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+
+
+/**
+ * <p>
+ * Abstract MapReduceApplication class.
+ * </p>
+ *
+ * @since 0.9.0
+ */
+@ApplicationAnnotation(name = "MapReduceExample")
+public abstract class MapReduceApplication<K1, V1, K2, V2> implements StreamingApplication
+{
+ Class<? extends InputFormat<K1, V1>> inputFormat;
+ Class<? extends Mapper<K1, V1, K2, V2>> mapClass;
+ Class<? extends Reducer<K2, V2, K2, V2>> reduceClass;
+ Class<? extends Reducer<K2, V2, K2, V2>> combineClass;
+
+ public Class<? extends Reducer<K2, V2, K2, V2>> getCombineClass()
+ {
+ return combineClass;
+ }
+
+ public void setCombineClass(Class<? extends Reducer<K2, V2, K2, V2>> combineClass)
+ {
+ this.combineClass = combineClass;
+ }
+
+ public void setInputFormat(Class<? extends InputFormat<K1, V1>> inputFormat)
+ {
+ this.inputFormat = inputFormat;
+ }
+
+ public Class<? extends Mapper<K1, V1, K2, V2>> getMapClass()
+ {
+ return mapClass;
+ }
+
+ public void setMapClass(Class<? extends Mapper<K1, V1, K2, V2>> mapClass)
+ {
+ this.mapClass = mapClass;
+ }
+
+ public Class<? extends Reducer<K2, V2, K2, V2>> getReduceClass()
+ {
+ return reduceClass;
+ }
+
+ public void setReduceClass(Class<? extends Reducer<K2, V2, K2, V2>> reduceClass)
+ {
+ this.reduceClass = reduceClass;
+ }
+
+
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ String configurationFilePath = conf.get(this.getClass().getSimpleName() + ".configFile", "");
+
+ MapOperator<K1, V1, K2, V2> inputOperator = dag.addOperator("Mapper", new MapOperator<K1, V1, K2, V2>());
+ inputOperator.setInputFormatClass(inputFormat);
+
+ String configFileName = null;
+ if (configurationFilePath != null && !configurationFilePath.isEmpty()) {
+ StringTokenizer configFileTokenizer = new StringTokenizer(configurationFilePath, "/");
+ configFileName = configFileTokenizer.nextToken();
+ while (configFileTokenizer.hasMoreTokens()) {
+ configFileName = configFileTokenizer.nextToken();
+ }
+ }
+
+ inputOperator.setMapClass(mapClass);
+ inputOperator.setConfigFile(configFileName);
+ inputOperator.setCombineClass(combineClass);
+
+ ReduceOperator<K2, V2, K2, V2> reduceOpr = dag.addOperator("Reducer", new ReduceOperator<K2, V2, K2, V2>());
+ reduceOpr.setReduceClass(reduceClass);
+ reduceOpr.setConfigFile(configFileName);
+
+ HdfsKeyValOutputOperator<K2, V2> console = dag.addOperator("Console", new HdfsKeyValOutputOperator<K2, V2>());
+
+ dag.addStream("Mapped-Output", inputOperator.output, reduceOpr.input);
+ dag.addStream("Mapper-Count", inputOperator.outputCount, reduceOpr.inputCount);
+ dag.addStream("Reduced-Output", reduceOpr.output, console.input);
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/NewWordCountApplication.java
----------------------------------------------------------------------
diff --git a/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/NewWordCountApplication.java b/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/NewWordCountApplication.java
new file mode 100644
index 0000000..db32b4a
--- /dev/null
+++ b/examples/mroperator/src/main/java/org/apache/apex/examples/mroperator/NewWordCountApplication.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.mroperator;
+
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.TextInputFormat;
+
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+
+/**
+ * <p>NewWordCountApplication class.</p>
+ *
+ * @since 0.9.0
+ */
+@ApplicationAnnotation(name = "WordCountExample")
+public class NewWordCountApplication extends MapReduceApplication<LongWritable, Text, Text, IntWritable>
+{
+
+ public void NewWordCountApplication()
+ {
+ setMapClass(WordCount.Map.class);
+ setReduceClass(WordCount.Reduce.class);
+ setCombineClass(WordCount.Reduce.class);
+ setInputFormat(TextInputFormat.class);
+ }
+}
[15/30] apex-malhar git commit: Renamed demos to examples. Packages
and artifactid names are changed as suggested.
Posted by th...@apache.org.
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/yahoofinance/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/demos/yahoofinance/src/main/resources/META-INF/properties.xml b/demos/yahoofinance/src/main/resources/META-INF/properties.xml
deleted file mode 100644
index 9186b98..0000000
--- a/demos/yahoofinance/src/main/resources/META-INF/properties.xml
+++ /dev/null
@@ -1,81 +0,0 @@
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<!--Configuration for chart demo -->
-<configuration>
- <property>
- <name>dt.attr.MASTER_MEMORY_MB</name>
- <value>1024</value>
- </property>
- <property>
- <name>dt.attr.STREAMING_WINDOW_SIZE_MILLIS</name>
- <value>1000</value>
- </property>
- <property>
- <name>dt.application.YahooFinanceDemoWithChart.operator.AverageChart.xAxisLabel</name>
- <value>TIME</value>
- </property>
- <property>
- <name>dt.application.YahooFinanceDemoWithChart.operator.StockTickInput.tickers</name>
- <value>IBM,GOOG,AAPL,YHOO</value>
- </property>
- <property>
- <name>dt.application.*.operator.*.attr.MEMORY_MB</name>
- <value>256</value>
- </property>
- <property>
- <name>dt.application.*.operator.*.attr.JVM_OPTIONS</name>
- <value>-Xmx128M</value>
- </property>
- <property>
- <name>dt.application.*.operator.*.port.*.attr.BUFFER_MEMORY_MB</name>
- <value>256</value>
- </property>
- <property>
- <name>dt.application.YahooFinanceDemo.operator.StockTickInput.tickers</name>
- <value>IBM,GOOG,AAPL,YHOO</value>
- </property>
- <property>
- <name>dt.application.YahooFinanceWithoutChartDemo.operator.StockTickInput.tickers</name>
- <value>IBM,GOOG,AAPL,YHOO</value>
- </property>
- <property>
- <name>dt.application.YahooFinanceDemoWithChart.operator.AverageChart.yAxisLabel</name>
- <value>PRICE</value>
- </property>
- <property>
- <name>dt.application.YahooFinanceDemoWithChart.operator.CandleStickChart.xAxisLabel</name>
- <value>TIME</value>
- </property>
- <property>
- <name>dt.application.YahooFinanceDemoWithChart.operator.CandleStickChart.yAxisLabel</name>
- <value>PRICE</value>
- </property>
- <property>
- <name>dt.application.YahooFinanceDemoWithChart.operator.AverageChart.attr.APPLICATION_WINDOW_COUNT
- </name>
- <value>5</value>
- </property>
- <property>
- <name>dt.application.YahooFinanceDemoWithChart.operator.AverageChart.attr.APPLICATION_WINDOW_COUNT
- </name>
- <value>5</value>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/yahoofinance/src/test/java/com/datatorrent/demos/yahoofinance/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/demos/yahoofinance/src/test/java/com/datatorrent/demos/yahoofinance/ApplicationTest.java b/demos/yahoofinance/src/test/java/com/datatorrent/demos/yahoofinance/ApplicationTest.java
deleted file mode 100644
index c038e61..0000000
--- a/demos/yahoofinance/src/test/java/com/datatorrent/demos/yahoofinance/ApplicationTest.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.yahoofinance;
-
-import org.junit.Test;
-import org.apache.hadoop.conf.Configuration;
-import com.datatorrent.api.LocalMode;
-
-/**
- * Run Yahoo Finance application demo.
- *
- */
-public class ApplicationTest
-{
-
- /**
- * This will run for ever.
- *
- * @throws Exception
- */
- @Test
- public void testApplication() throws Exception
- {
- LocalMode lma = LocalMode.newInstance();
- new YahooFinanceApplication().populateDAG(lma.getDAG(), new Configuration(false));
- LocalMode.Controller lc = lma.getController();
- lc.run(10000);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/yahoofinance/src/test/java/com/datatorrent/demos/yahoofinance/ApplicationWithDerbySQLTest.java
----------------------------------------------------------------------
diff --git a/demos/yahoofinance/src/test/java/com/datatorrent/demos/yahoofinance/ApplicationWithDerbySQLTest.java b/demos/yahoofinance/src/test/java/com/datatorrent/demos/yahoofinance/ApplicationWithDerbySQLTest.java
deleted file mode 100644
index 7b134f5..0000000
--- a/demos/yahoofinance/src/test/java/com/datatorrent/demos/yahoofinance/ApplicationWithDerbySQLTest.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.yahoofinance;
-
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.conf.Configuration;
-import com.datatorrent.api.LocalMode;
-
-/**
- *
- */
-public class ApplicationWithDerbySQLTest
-{
- private final transient Logger LOG = LoggerFactory.getLogger(ApplicationWithDerbySQLTest.class);
- public ApplicationWithDerbySQLTest()
- {
- }
-
- @Test
- public void testSomeMethod() throws Exception
- {
- LocalMode lma = LocalMode.newInstance();
- new ApplicationWithDerbySQL().populateDAG(lma.getDAG(), new Configuration(false));
- LocalMode.Controller lc = lma.getController();
-
- long start = System.currentTimeMillis();
- lc.run();
- long end = System.currentTimeMillis();
- long time = end - start;
- LOG.debug("Test used " + time + " ms");
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/yahoofinance/src/test/resources/alert_create.json
----------------------------------------------------------------------
diff --git a/demos/yahoofinance/src/test/resources/alert_create.json b/demos/yahoofinance/src/test/resources/alert_create.json
deleted file mode 100644
index 3059145..0000000
--- a/demos/yahoofinance/src/test/resources/alert_create.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "name":"alertName",
- "streamName":"yahooFinance.outputPort",
- "filter": {
- "class": "com.datatorrent.lib.util.JavaScriptFilterOperator",
- "properties": {
- "setupScript":"function f() { return s0 == \"AAPL\" && l1 > 508 }",
- "functionName":"f"
- }
- },
- "escalation": {
- "class": "com.datatorrent.lib.util.AlertEscalationOperator",
- "properties": {
- "alertInterval":"5000",
- "timeout":"10000"
- }
- },
- "actions": [{
- "outputPort":"alert",
- "inputPort":"input",
- "class":"com.datatorrent.lib.io.ConsoleOutputOperator"
- }],
- "saveAs":"firstAlert"
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/yahoofinance/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/demos/yahoofinance/src/test/resources/log4j.properties b/demos/yahoofinance/src/test/resources/log4j.properties
deleted file mode 100644
index cf0d19e..0000000
--- a/demos/yahoofinance/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,43 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-log4j.rootLogger=DEBUG,CONSOLE
-
-log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
-log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
-log4j.appender.CONSOLE.threshold=${test.log.console.threshold}
-test.log.console.threshold=DEBUG
-
-log4j.appender.RFA=org.apache.log4j.RollingFileAppender
-log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
-log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
-log4j.appender.RFA.File=/tmp/app.log
-
-# to enable, add SYSLOG to rootLogger
-log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender
-log4j.appender.SYSLOG.syslogHost=127.0.0.1
-log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout
-log4j.appender.SYSLOG.layout.conversionPattern=${dt.cid} %-5p [%t] %c{2} %x - %m%n
-log4j.appender.SYSLOG.Facility=LOCAL1
-
-log4j.logger.org=info
-#log4j.logger.org.apache.commons.beanutils=warn
-log4j.logger.com.datatorrent=debug
-log4j.logger.org.apache.apex=debug
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/docs/CodingConventionsAndStyle.md
----------------------------------------------------------------------
diff --git a/docs/CodingConventionsAndStyle.md b/docs/CodingConventionsAndStyle.md
index 154f1de..baf2943 100644
--- a/docs/CodingConventionsAndStyle.md
+++ b/docs/CodingConventionsAndStyle.md
@@ -1,7 +1,7 @@
Coding Conventions and Style
==============================
-Malhar GitHub repository contains operator library and demos built on top of the Apex platform. The code is open source, viewable and downloadable by all. Anyone can make code submissions to the repository to add new features or fix bugs. The process to do so is to first make a personal fork of the repository, make changes in the fork and then generate a pull request with the changes against the Malhar repository.
+Malhar GitHub repository contains operator library and examples built on top of the Apex platform. The code is open source, viewable and downloadable by all. Anyone can make code submissions to the repository to add new features or fix bugs. The process to do so is to first make a personal fork of the repository, make changes in the fork and then generate a pull request with the changes against the Malhar repository.
Malhar administrators look at pull requests regularly and merge them into the repository. The pull requests have to follow certain guidelines in order to minimize the possibility of issues and problems arising from the merge, to keep the code maintainable going forward and to keep the licensing. The guidelines are as follows
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/docs/index.md
----------------------------------------------------------------------
diff --git a/docs/index.md b/docs/index.md
index 7b6a441..2d6bc46 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -1,7 +1,7 @@
Apache Apex Malhar
================================================================================
-Apache Apex Malhar is an open source operator and codec library that can be used with the [Apache Apex](http://apex.apache.org/) platform to build real-time streaming applications. Enabling users to extract value quickly, Malhar operators help get data in, analyze it in real-time, and get data out of Hadoop. In addition to the operators, the library contains a number of demos applications, demonstrating operator features and capabilities.
+Apache Apex Malhar is an open source operator and codec library that can be used with the [Apache Apex](http://apex.apache.org/) platform to build real-time streaming applications. Enabling users to extract value quickly, Malhar operators help get data in, analyze it in real-time, and get data out of Hadoop. In addition to the operators, the library contains a number of example applications, demonstrating operator features and capabilities.
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/docs/operators/windowedOperator.md
----------------------------------------------------------------------
diff --git a/docs/operators/windowedOperator.md b/docs/operators/windowedOperator.md
index c0ac142..22012bf 100644
--- a/docs/operators/windowedOperator.md
+++ b/docs/operators/windowedOperator.md
@@ -247,7 +247,7 @@ The `WindowedMergeOperator` has its own watermark. Its watermark timestamp is th
## Usage Examples
-For an example usage of the `WindowedOperator` via the High level API, click [here](https://github.com/apache/apex-malhar/blob/master/demos/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/WindowedWordCount.java).
+For an example usage of the `WindowedOperator` via the High level API, click [here](https://github.com/apache/apex-malhar/blob/master/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/WindowedWordCount.java).
For an example usage of the `WindowedOperator` via the DAG level API, click [here](https://github.com/apache/apex-malhar/blob/master/library/src/test/java/org/apache/apex/malhar/lib/window/sample/wordcount/Application.java).
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/distributedistinct/pom.xml
----------------------------------------------------------------------
diff --git a/examples/distributedistinct/pom.xml b/examples/distributedistinct/pom.xml
new file mode 100644
index 0000000..510ea68
--- /dev/null
+++ b/examples/distributedistinct/pom.xml
@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>malhar-examples-distributedistinct</artifactId>
+ <packaging>jar</packaging>
+
+ <name>Apache Apex Malhar Distributed Distinct Example</name>
+ <description></description>
+
+ <parent>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-examples</artifactId>
+ <version>3.7.0-SNAPSHOT</version>
+ </parent>
+
+ <properties>
+ <skipTests>true</skipTests>
+ </properties>
+
+</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/distributedistinct/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/examples/distributedistinct/src/assemble/appPackage.xml b/examples/distributedistinct/src/assemble/appPackage.xml
new file mode 100644
index 0000000..4138cf2
--- /dev/null
+++ b/examples/distributedistinct/src/assemble/appPackage.xml
@@ -0,0 +1,59 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+ <id>appPackage</id>
+ <formats>
+ <format>jar</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>${basedir}/target/</directory>
+ <outputDirectory>/app</outputDirectory>
+ <includes>
+ <include>${project.artifactId}-${project.version}.jar</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/target/deps</directory>
+ <outputDirectory>/lib</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/site/conf</directory>
+ <outputDirectory>/conf</outputDirectory>
+ <includes>
+ <include>*.xml</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/META-INF</directory>
+ <outputDirectory>/META-INF</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/app</directory>
+ <outputDirectory>/app</outputDirectory>
+ </fileSet>
+ </fileSets>
+
+</assembly>
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/distributedistinct/src/main/java/org/apache/apex/examples/distributeddistinct/Application.java
----------------------------------------------------------------------
diff --git a/examples/distributedistinct/src/main/java/org/apache/apex/examples/distributeddistinct/Application.java b/examples/distributedistinct/src/main/java/org/apache/apex/examples/distributeddistinct/Application.java
new file mode 100644
index 0000000..8eb8f2d
--- /dev/null
+++ b/examples/distributedistinct/src/main/java/org/apache/apex/examples/distributeddistinct/Application.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.distributeddistinct;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.lib.algo.UniqueValueCount;
+import com.datatorrent.lib.io.ConsoleOutputOperator;
+import com.datatorrent.lib.stream.Counter;
+import com.datatorrent.lib.stream.StreamDuplicater;
+import com.datatorrent.lib.util.KeyValPair;
+
+/**
+ * This application demonstrates the UniqueValueCount operator. It uses an input operator which generates random key
+ * value pairs and simultaneously emits them to the UniqueValueCount operator and keeps track of the number of unique
+ * values per key to emit to the verifier.
+ *
+ * @since 1.0.4
+ */
+@ApplicationAnnotation(name = "ValueCount")
+public class Application implements StreamingApplication
+{
+
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ RandomKeyValGenerator randGen = dag.addOperator("RandomGenerator", new RandomKeyValGenerator());
+ UniqueValueCount<Integer> valCount = dag.addOperator("UniqueCounter", new UniqueValueCount<Integer>());
+ ConsoleOutputOperator consOut = dag.addOperator("Console", new ConsoleOutputOperator());
+ StreamDuplicater<KeyValPair<Integer, Integer>> dup = dag.addOperator("Duplicator", new StreamDuplicater<KeyValPair<Integer, Integer>>());
+ CountVerifier verifier = dag.addOperator("Verifier", new CountVerifier());
+ ConsoleOutputOperator successOutput = dag.addOperator("Success", new ConsoleOutputOperator());
+ successOutput.setStringFormat("Success %d");
+ ConsoleOutputOperator failureOutput = dag.addOperator("Failure", new ConsoleOutputOperator());
+ failureOutput.setStringFormat("Failure %d");
+
+ Counter successcounter = dag.addOperator("SuccessCounter", new Counter());
+ Counter failurecounter = dag.addOperator("FailureCounter", new Counter());
+
+ dag.addStream("Events", randGen.outport, valCount.input);
+ dag.addStream("Duplicates", valCount.output, dup.data);
+ dag.addStream("Unverified", dup.out1, verifier.recIn);
+ dag.addStream("EventCount", randGen.verport, verifier.trueIn);
+ dag.addStream("Verified", verifier.successPort, successcounter.input);
+ dag.addStream("Failed", verifier.failurePort, failurecounter.input);
+ dag.addStream("SuccessCount", successcounter.output, successOutput.input);
+ dag.addStream("FailedCount", failurecounter.output, failureOutput.input);
+ dag.addStream("Output", dup.out2, consOut.input);
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/distributedistinct/src/main/java/org/apache/apex/examples/distributeddistinct/CountVerifier.java
----------------------------------------------------------------------
diff --git a/examples/distributedistinct/src/main/java/org/apache/apex/examples/distributeddistinct/CountVerifier.java b/examples/distributedistinct/src/main/java/org/apache/apex/examples/distributeddistinct/CountVerifier.java
new file mode 100644
index 0000000..380ed4e
--- /dev/null
+++ b/examples/distributedistinct/src/main/java/org/apache/apex/examples/distributeddistinct/CountVerifier.java
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.distributeddistinct;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import com.datatorrent.api.Context.OperatorContext;
+import com.datatorrent.api.DefaultInputPort;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.Operator;
+import com.datatorrent.api.annotation.OutputPortFieldAnnotation;
+import com.datatorrent.lib.util.KeyValPair;
+
+/**
+ * An operator that takes two streams of Integer to Integer KeyValPairs and verifies that the two streams output the
+ * same KeyValPairs within a given window.
+ *
+ * @since 1.0.4
+ */
+public class CountVerifier implements Operator
+{
+ Map<Integer, Integer> trueCount = new HashMap<Integer, Integer>();
+ Map<Integer, Integer> receivedCount = new HashMap<Integer, Integer>();
+
+ public final transient DefaultInputPort<KeyValPair<Integer, Integer>> trueIn = new DefaultInputPort<KeyValPair<Integer, Integer>>()
+ {
+ @Override
+ public void process(KeyValPair<Integer, Integer> tuple)
+ {
+ trueCount.put(tuple.getKey(), tuple.getValue());
+ }
+ };
+
+ public final transient DefaultInputPort<KeyValPair<Integer, Integer>> recIn = new DefaultInputPort<KeyValPair<Integer, Integer>>()
+ {
+ @Override
+ public void process(KeyValPair<Integer, Integer> tuple)
+ {
+ receivedCount.put(tuple.getKey(), tuple.getValue());
+ }
+ };
+
+ @OutputPortFieldAnnotation(optional = true)
+ public final transient DefaultOutputPort<Integer> successPort = new DefaultOutputPort<Integer>();
+ @OutputPortFieldAnnotation(optional = true)
+ public final transient DefaultOutputPort<Integer> failurePort = new DefaultOutputPort<Integer>();
+
+ @Override
+ public void setup(OperatorContext arg0)
+ {
+
+ }
+
+ @Override
+ public void teardown()
+ {
+
+ }
+
+ @Override
+ public void beginWindow(long windowID)
+ {
+ }
+
+ /**
+ * Checks that the key to value pairs are the same and counts the number of pairs that are different. If there are
+ * failures, it will emit the number of failures to the failure port. Otherwise, it will emit the number of keys to
+ * the success port.
+ */
+ @Override
+ public void endWindow()
+ {
+ int failureCount = 0;
+ for (Map.Entry<Integer, Integer> e : receivedCount.entrySet()) {
+ Integer key = e.getKey();
+ if (!trueCount.get(key).equals(e.getValue())) {
+ failureCount++;
+ }
+ }
+ if (failureCount != 0) {
+ failurePort.emit(failureCount);
+ } else {
+ successPort.emit(trueCount.size());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/distributedistinct/src/main/java/org/apache/apex/examples/distributeddistinct/IntegerUniqueValueCountAppender.java
----------------------------------------------------------------------
diff --git a/examples/distributedistinct/src/main/java/org/apache/apex/examples/distributeddistinct/IntegerUniqueValueCountAppender.java b/examples/distributedistinct/src/main/java/org/apache/apex/examples/distributeddistinct/IntegerUniqueValueCountAppender.java
new file mode 100644
index 0000000..229eb32
--- /dev/null
+++ b/examples/distributedistinct/src/main/java/org/apache/apex/examples/distributeddistinct/IntegerUniqueValueCountAppender.java
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.distributeddistinct;
+
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.HashSet;
+import java.util.Set;
+
+import com.datatorrent.lib.algo.UniqueValueCount;
+import com.datatorrent.lib.util.KeyValPair;
+
+/**
+ * This operator demonstrates {@link UniqueValueCountAppender} given that the keys and values of the preceding {@link UniqueValueCount} operator
+ * are both integers. <br/>
+ * It will keep track of the number of all the unique values emitted per key since the application starts.
+ *
+ * @since 1.0.4
+ */
+public class IntegerUniqueValueCountAppender extends UniqueValueCountAppender<Integer>
+{
+ @Override
+ public Object processResultSet(ResultSet resultSet)
+ {
+ Set<Integer> valSet = new HashSet<Integer>();
+ try {
+ while (resultSet.next()) {
+ valSet.add(resultSet.getInt(1));
+ }
+ return valSet;
+ } catch (SQLException e) {
+ throw new RuntimeException("while processing the result set", e);
+ }
+ }
+
+ @Override
+ protected void prepareGetStatement(PreparedStatement getStatement, Object key) throws SQLException
+ {
+ getStatement.setInt(1, (Integer)key);
+ }
+
+ @Override
+ protected void preparePutStatement(PreparedStatement putStatement, Object key, Object value) throws SQLException
+ {
+ @SuppressWarnings("unchecked")
+ Set<Integer> valueSet = (Set<Integer>)value;
+ for (Integer val : valueSet) {
+ @SuppressWarnings("unchecked")
+ Set<Integer> currentVals = (Set<Integer>)get(key);
+ if (!currentVals.contains(val)) {
+ batch = true;
+ putStatement.setInt(1, (Integer)key);
+ putStatement.setInt(2, val);
+ putStatement.setLong(3, windowID);
+ putStatement.addBatch();
+ }
+ }
+ }
+
+ @Override
+ public void endWindow()
+ {
+ try {
+ Statement stmt = store.getConnection().createStatement();
+ String keySetQuery = "SELECT DISTINCT col1 FROM " + tableName;
+ ResultSet resultSet = stmt.executeQuery(keySetQuery);
+ while (resultSet.next()) {
+ int val = resultSet.getInt(1);
+ @SuppressWarnings("unchecked")
+ Set<Integer> valSet = (Set<Integer>)cacheManager.get(val);
+ output.emit(new KeyValPair<Object, Object>(val, valSet.size()));
+ }
+ } catch (SQLException e) {
+ throw new RuntimeException("While emitting tuples", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/distributedistinct/src/main/java/org/apache/apex/examples/distributeddistinct/RandomKeyValGenerator.java
----------------------------------------------------------------------
diff --git a/examples/distributedistinct/src/main/java/org/apache/apex/examples/distributeddistinct/RandomKeyValGenerator.java b/examples/distributedistinct/src/main/java/org/apache/apex/examples/distributeddistinct/RandomKeyValGenerator.java
new file mode 100644
index 0000000..b8e04be
--- /dev/null
+++ b/examples/distributedistinct/src/main/java/org/apache/apex/examples/distributeddistinct/RandomKeyValGenerator.java
@@ -0,0 +1,186 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.distributeddistinct;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.datatorrent.api.Context.OperatorContext;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.InputOperator;
+import com.datatorrent.api.annotation.OutputPortFieldAnnotation;
+import com.datatorrent.lib.util.KeyHashValPair;
+import com.datatorrent.lib.util.KeyValPair;
+
+/**
+ * Generates random KeyValPairs and optionally, keeps track of the number of unique values per pair to emit to the
+ * verification port.
+ *
+ * @since 1.0.4
+ */
+public class RandomKeyValGenerator implements InputOperator
+{
+
+ protected int numKeys = 50;
+ protected int numVals = 1000;
+ protected int tupleBlast = 1000;
+ protected Map<Integer, Set<Integer>> valhistory = new HashMap<Integer, Set<Integer>>();
+ private Random rand = new Random();
+ private boolean once;
+ private boolean clearHistory;
+ @OutputPortFieldAnnotation(optional = false)
+ public transient DefaultOutputPort<KeyValPair<Integer, Object>> outport = new DefaultOutputPort<KeyValPair<Integer, Object>>();
+
+ @OutputPortFieldAnnotation(optional = true)
+ public transient DefaultOutputPort<KeyValPair<Integer, Integer>> verport = new DefaultOutputPort<KeyValPair<Integer, Integer>>();
+
+ /**
+ * Ensures that the generator emits KeyValPairs once per window
+ */
+ @Override
+ public void beginWindow(long l)
+ {
+ once = false;
+ }
+
+ /**
+ * Emits the total count of unique values per key as KeyHashValPairs to the verification port
+ */
+ @Override
+ public void endWindow()
+ {
+ if (verport.isConnected()) {
+ for (Map.Entry<Integer, Set<Integer>> e : valhistory.entrySet()) {
+ verport.emit(new KeyHashValPair<Integer, Integer>(e.getKey(), e.getValue().size()));
+ }
+ }
+ if (clearHistory) {
+ valhistory.clear();
+ }
+ }
+
+ @Override
+ public void setup(OperatorContext arg0)
+ {
+
+ }
+
+ @Override
+ public void teardown()
+ {
+
+ }
+
+ /**
+ * Emits random KeyValPairs and keeps track of the unique values per key.
+ */
+ @Override
+ public void emitTuples()
+ {
+ if (!once) {
+ int key;
+ int val;
+ for (int i = 0; i < tupleBlast; i++) {
+ key = rand.nextInt(numKeys);
+ val = rand.nextInt(numVals);
+ outport.emit(new KeyValPair<Integer, Object>(key, val));
+ if (verport.isConnected()) {
+ Set<Integer> count = valhistory.get(key);
+ if (count == null) {
+ Set<Integer> tempset = new HashSet<Integer>();
+ tempset.add(val);
+ valhistory.put(key, tempset);
+ LOG.debug("key {} val {}", key, tempset);
+ } else if (!valhistory.get(key).contains(val)) {
+ valhistory.get(key).add(val);
+ }
+ }
+ }
+ once = true;
+ }
+ }
+
+ /**
+ * @return the number of possible keys
+ */
+ public int getNumKeys()
+ {
+ return numKeys;
+ }
+
+ /**
+ * Sets the number of possible keys to numKeys
+ *
+ * @param numKeys
+ * the new number of possible keys
+ */
+ public void setNumKeys(int numKeys)
+ {
+ this.numKeys = numKeys;
+ }
+
+ /**
+ * Returns the number of possible values that can be emitted
+ *
+ * @return the number of possible values that can be emitted
+ */
+ public int getNumVals()
+ {
+ return numVals;
+ }
+
+ /**
+ * Sets the number of possible values that can be emitted to numVals
+ *
+ * @param numVals
+ * the number of possible values that can be emitted
+ */
+ public void setNumVals(int numVals)
+ {
+ this.numVals = numVals;
+ }
+
+ /**
+ * Sets the number of KeyValPairs to be emitted to tupleBlast
+ *
+ * @param tupleBlast
+ * the new number of KeyValPairs to be emitted
+ */
+ public void setTupleBlast(int tupleBlast)
+ {
+ this.tupleBlast = tupleBlast;
+ }
+
+ /**
+ * tuple blast
+ * @return
+ */
+ public int getTupleBlast()
+ {
+ return tupleBlast;
+ }
+
+ private static final Logger LOG = LoggerFactory.getLogger(RandomKeyValGenerator.class);
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/distributedistinct/src/main/java/org/apache/apex/examples/distributeddistinct/StatefulApplication.java
----------------------------------------------------------------------
diff --git a/examples/distributedistinct/src/main/java/org/apache/apex/examples/distributeddistinct/StatefulApplication.java b/examples/distributedistinct/src/main/java/org/apache/apex/examples/distributeddistinct/StatefulApplication.java
new file mode 100644
index 0000000..550000e
--- /dev/null
+++ b/examples/distributedistinct/src/main/java/org/apache/apex/examples/distributeddistinct/StatefulApplication.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.distributeddistinct;
+
+import java.io.Serializable;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.Context;
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.lib.algo.UniqueValueCount;
+import com.datatorrent.lib.algo.UniqueValueCount.InternalCountOutput;
+import com.datatorrent.lib.codec.KryoSerializableStreamCodec;
+import com.datatorrent.lib.io.ConsoleOutputOperator;
+
+/**
+ * This application demonstrates the UniqueValueCount operator. It uses an input operator which generates random key
+ * value pairs and emits them to the UniqueValueCount operator which keeps track of the unique values per window. It
+ * then emits the values to the StatefulUniqueCount which uses a combination of a cache and database to keep track of
+ * the overall unique values and outputs the resulting unique value count to the ConsoleOutputOperator.
+ *
+ * @since 1.0.4
+ */
+@ApplicationAnnotation(name = "StatefulDistinctCount")
+public class StatefulApplication implements StreamingApplication
+{
+ @SuppressWarnings("unchecked")
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ RandomKeyValGenerator randGen = dag.addOperator("RandomGenerator", new RandomKeyValGenerator());
+ UniqueValueCount<Integer> valCount = dag.addOperator("UniqueCounter", new UniqueValueCount<Integer>());
+ ConsoleOutputOperator consOut = dag.addOperator("Console", new ConsoleOutputOperator());
+ IntegerUniqueValueCountAppender uniqueUnifier = dag.addOperator("StatefulUniqueCounter", new IntegerUniqueValueCountAppender());
+ dag.getOperatorMeta("StatefulUniqueCounter").getMeta(uniqueUnifier.input).getAttributes().put(Context.PortContext.STREAM_CODEC, new KeyBasedStreamCodec());
+
+ @SuppressWarnings("rawtypes")
+ DefaultOutputPort valOut = valCount.output;
+ @SuppressWarnings("rawtypes")
+ DefaultOutputPort uniqueOut = uniqueUnifier.output;
+
+ dag.addStream("Events", randGen.outport, valCount.input);
+ dag.addStream("Unified", valOut, uniqueUnifier.input);
+ dag.addStream("Result", uniqueOut, consOut.input);
+ }
+
+ public static class KeyBasedStreamCodec extends KryoSerializableStreamCodec<InternalCountOutput<Integer>> implements Serializable
+ {
+ @Override
+ public int getPartition(InternalCountOutput<Integer> t)
+ {
+ return t.getKey().hashCode();
+ }
+
+ private static final long serialVersionUID = 201407231527L;
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/distributedistinct/src/main/java/org/apache/apex/examples/distributeddistinct/UniqueValueCountAppender.java
----------------------------------------------------------------------
diff --git a/examples/distributedistinct/src/main/java/org/apache/apex/examples/distributeddistinct/UniqueValueCountAppender.java b/examples/distributedistinct/src/main/java/org/apache/apex/examples/distributeddistinct/UniqueValueCountAppender.java
new file mode 100644
index 0000000..c81f7b8
--- /dev/null
+++ b/examples/distributedistinct/src/main/java/org/apache/apex/examples/distributeddistinct/UniqueValueCountAppender.java
@@ -0,0 +1,236 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.distributeddistinct;
+
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
+
+import javax.annotation.Nonnull;
+import javax.validation.constraints.Min;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+
+import com.datatorrent.api.Context;
+import com.datatorrent.api.DefaultPartition;
+import com.datatorrent.api.Partitioner;
+import com.datatorrent.lib.algo.UniqueValueCount;
+import com.datatorrent.lib.algo.UniqueValueCount.InternalCountOutput;
+import com.datatorrent.lib.db.jdbc.JDBCLookupCacheBackedOperator;
+import com.datatorrent.netlet.util.DTThrowable;
+
+/**
+ * <p>
+ * This operator supplements the {@link UniqueValueCount} operator by making it state-full.<br/>
+ * It helps to track unique values through out the lifetime of the application.
+ * </p>
+ *
+ * <p>
+ * The operator counts the number of values emitted per key by storing previously
+ * counted values in both a transient cache and in a persistent database. <br/>
+ * In case of a rollback, it will erase all values committed to the database
+ * in the windows greater than the activation window, then re-enter them as needed to keep it stateful.<br/>
+ * This operator, when appended to {@link UniqueValueCount} will keep track of the
+ * unique values emitted since the start of the application.
+ *
+ * @since 1.0.4
+ */
+public abstract class UniqueValueCountAppender<V> extends JDBCLookupCacheBackedOperator<InternalCountOutput<V>> implements Partitioner<UniqueValueCountAppender<V>>
+{
+ protected Set<Integer> partitionKeys;
+ protected int partitionMask;
+ protected transient long windowID;
+ protected transient boolean batch;
+ @Min(1)
+ private int partitionCount = 1;
+
+ public UniqueValueCountAppender()
+
+ {
+ partitionKeys = Sets.newHashSet(0);
+ partitionMask = 0;
+ }
+
+ public void setPartitionCount(int partitionCount)
+ {
+ this.partitionCount = partitionCount;
+ }
+
+ public int getPartitionCount()
+ {
+ return partitionCount;
+ }
+
+ @Override
+ public void setup(Context.OperatorContext context)
+ {
+ super.setup(context);
+ LOGGER.debug("store properties {} {}", store.getDatabaseDriver(), store.getDatabaseUrl());
+ LOGGER.debug("table name {}", tableName);
+ windowID = context.getValue(Context.OperatorContext.ACTIVATION_WINDOW_ID);
+ try {
+ ResultSet resultSet = store.getConnection().createStatement().executeQuery("SELECT col1 FROM " + tableName + " WHERE col3 >= " + windowID);
+ PreparedStatement deleteStatement = store.getConnection().prepareStatement("DELETE FROM " + tableName + " WHERE col3 >= " + windowID + " AND col1 = ?");
+
+ Set<Object> deletedKeys = Sets.newHashSet();
+ while (resultSet.next()) {
+ Object key = resultSet.getObject(1);
+ if (partitionKeys.contains((key.hashCode() & partitionMask)) && !deletedKeys.contains(key)) {
+ deletedKeys.add(key);
+ deleteStatement.setObject(1, key);
+ deleteStatement.executeUpdate();
+ }
+ }
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ protected void processTuple(InternalCountOutput<V> tuple)
+ {
+
+ Object key = getKeyFromTuple(tuple);
+ @SuppressWarnings("unchecked")
+ Set<Object> values = (Set<Object>)cacheManager.get(key);
+ if (values == null) {
+ values = Sets.newHashSet();
+ }
+ values.addAll(tuple.getInternalSet());
+ cacheManager.put(key, values);
+ }
+
+ @Override
+ protected String fetchInsertQuery()
+ {
+ return "INSERT INTO " + tableName + " (col1, col2, col3) VALUES (?, ?, ?)";
+ }
+
+ @Override
+ protected String fetchGetQuery()
+ {
+ return "select col2 from " + tableName + " where col1 = ?";
+ }
+
+ @Override
+ public Map<Object, Object> loadInitialData()
+ {
+ return null;
+ }
+
+ @Override
+ public void put(@Nonnull Object key, @Nonnull Object value)
+ {
+ try {
+ batch = false;
+ preparePutStatement(putStatement, key, value);
+ if (batch) {
+ putStatement.executeBatch();
+ putStatement.clearBatch();
+ }
+ } catch (SQLException e) {
+ throw new RuntimeException("while executing insert", e);
+ }
+ }
+
+ @Override
+ public void teardown()
+ {
+
+ }
+
+ @Override
+ public void beginWindow(long windowID)
+ {
+ this.windowID = windowID;
+ }
+
+ @Override
+ protected Object getKeyFromTuple(InternalCountOutput<V> tuple)
+ {
+ return tuple.getKey();
+ }
+
+ @Override
+ public void putAll(Map<Object, Object> m)
+ {
+ throw new UnsupportedOperationException("not supported");
+ }
+
+ @Override
+ public void remove(Object key)
+ {
+ throw new UnsupportedOperationException("not supported");
+ }
+
+ /**
+ * Assigns the partitions according to certain key values and keeps track of the
+ * keys that each partition will be processing so that in the case of a
+ * rollback, each partition will only clear the data that it is responsible for.
+ */
+ @Override
+ public Collection<com.datatorrent.api.Partitioner.Partition<UniqueValueCountAppender<V>>> definePartitions(Collection<com.datatorrent.api.Partitioner.Partition<UniqueValueCountAppender<V>>> partitions, PartitioningContext context)
+ {
+ final int finalCapacity = DefaultPartition.getRequiredPartitionCount(context, this.partitionCount);
+ UniqueValueCountAppender<V> anOldOperator = partitions.iterator().next().getPartitionedInstance();
+ partitions.clear();
+
+ Collection<Partition<UniqueValueCountAppender<V>>> newPartitions = Lists.newArrayListWithCapacity(finalCapacity);
+
+ for (int i = 0; i < finalCapacity; i++) {
+ try {
+ @SuppressWarnings("unchecked")
+ UniqueValueCountAppender<V> statefulUniqueCount = this.getClass().newInstance();
+ DefaultPartition<UniqueValueCountAppender<V>> partition = new DefaultPartition<UniqueValueCountAppender<V>>(statefulUniqueCount);
+ newPartitions.add(partition);
+ } catch (Throwable cause) {
+ DTThrowable.rethrow(cause);
+ }
+ }
+
+ DefaultPartition.assignPartitionKeys(Collections.unmodifiableCollection(newPartitions), input);
+ int lPartitionMask = newPartitions.iterator().next().getPartitionKeys().get(input).mask;
+
+ for (Partition<UniqueValueCountAppender<V>> statefulUniqueCountPartition : newPartitions) {
+ UniqueValueCountAppender<V> statefulUniqueCountInstance = statefulUniqueCountPartition.getPartitionedInstance();
+
+ statefulUniqueCountInstance.partitionKeys = statefulUniqueCountPartition.getPartitionKeys().get(input).partitions;
+ statefulUniqueCountInstance.partitionMask = lPartitionMask;
+ statefulUniqueCountInstance.store = anOldOperator.store;
+ statefulUniqueCountInstance.tableName = anOldOperator.tableName;
+ statefulUniqueCountInstance.cacheManager = anOldOperator.cacheManager;
+ }
+ return newPartitions;
+ }
+
+ @Override
+ public void partitioned(Map<Integer, com.datatorrent.api.Partitioner.Partition<UniqueValueCountAppender<V>>> partitions)
+ {
+ }
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(UniqueValueCountAppender.class);
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/distributedistinct/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/examples/distributedistinct/src/main/resources/META-INF/properties.xml b/examples/distributedistinct/src/main/resources/META-INF/properties.xml
new file mode 100644
index 0000000..8742328
--- /dev/null
+++ b/examples/distributedistinct/src/main/resources/META-INF/properties.xml
@@ -0,0 +1,29 @@
+<?xml version="1.0"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<configuration>
+ <!--
+ <property>
+ <name>dt.application.{appName}.operator.{opName}.prop.{propName}</name>
+ <value>some-default-value (if value is not specified, it is required from the user or custom config when launching)</value>
+ </property>
+ -->
+</configuration>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/distributedistinct/src/test/java/org/apache/apex/examples/distributeddistinct/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/distributedistinct/src/test/java/org/apache/apex/examples/distributeddistinct/ApplicationTest.java b/examples/distributedistinct/src/test/java/org/apache/apex/examples/distributeddistinct/ApplicationTest.java
new file mode 100644
index 0000000..4861051
--- /dev/null
+++ b/examples/distributedistinct/src/test/java/org/apache/apex/examples/distributeddistinct/ApplicationTest.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.distributeddistinct;
+
+import org.junit.Test;
+
+import com.datatorrent.api.LocalMode;
+
+public class ApplicationTest
+{
+ @Test
+ public void testSomeMethod() throws Exception
+ {
+ LocalMode.runApp(new Application(), 15000);
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/distributedistinct/src/test/java/org/apache/apex/examples/distributeddistinct/DistributedDistinctTest.java
----------------------------------------------------------------------
diff --git a/examples/distributedistinct/src/test/java/org/apache/apex/examples/distributeddistinct/DistributedDistinctTest.java b/examples/distributedistinct/src/test/java/org/apache/apex/examples/distributeddistinct/DistributedDistinctTest.java
new file mode 100644
index 0000000..3ba39d6
--- /dev/null
+++ b/examples/distributedistinct/src/test/java/org/apache/apex/examples/distributeddistinct/DistributedDistinctTest.java
@@ -0,0 +1,200 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.distributeddistinct;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Properties;
+import java.util.Set;
+
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.datatorrent.api.Context.OperatorContext;
+import com.datatorrent.api.DAG;
+import com.datatorrent.lib.algo.UniqueValueCount.InternalCountOutput;
+import com.datatorrent.lib.helper.OperatorContextTestHelper;
+import com.datatorrent.lib.helper.OperatorContextTestHelper.TestIdOperatorContext;
+
+/**
+ * Test for {@link IntegerUniqueValueCountAppender} and {@link UniqueValueCountAppender}
+ */
+public class DistributedDistinctTest
+{
+ private static final Logger logger = LoggerFactory.getLogger(DistributedDistinctTest.class);
+
+ private static final String APP_ID = "DistributedDistinctTest";
+ private static final int OPERATOR_ID = 0;
+
+ public static final String INMEM_DB_URL = "jdbc:hsqldb:mem:test;sql.syntax_mys=true";
+ public static final String INMEM_DB_DRIVER = "org.hsqldb.jdbc.JDBCDriver";
+ public static final String TABLE_NAME = "Test_Lookup_Cache";
+
+ private static IntegerUniqueValueCountAppender valueCounter;
+ private static String applicationPath;
+
+ @Test
+ public void testProcess() throws Exception
+ {
+ insertValues();
+ Statement stmt = valueCounter.getStore().getConnection().createStatement();
+
+ ResultSet resultSet = stmt.executeQuery("SELECT col2 FROM " + TABLE_NAME + " WHERE col1 = 1");
+ ArrayList<Integer> answersOne = new ArrayList<Integer>();
+ for (int i = 1; i < 16; i++) {
+ answersOne.add(i);
+ }
+ Assert.assertEquals(answersOne, processResult(resultSet));
+
+ resultSet = stmt.executeQuery("SELECT col2 FROM " + TABLE_NAME + " WHERE col1 = 2");
+ ArrayList<Integer> answersTwo = new ArrayList<Integer>();
+ answersTwo.add(3);
+ answersTwo.add(6);
+ answersTwo.add(9);
+ for (int i = 11; i < 21; i++) {
+ answersTwo.add(i);
+ }
+ Assert.assertEquals(answersTwo, processResult(resultSet));
+
+ resultSet = stmt.executeQuery("SELECT col2 FROM " + TABLE_NAME + " WHERE col1 = 3");
+ ArrayList<Integer> answersThree = new ArrayList<Integer>();
+ answersThree.add(2);
+ answersThree.add(4);
+ answersThree.add(6);
+ answersThree.add(8);
+ answersThree.add(10);
+ for (int i = 11; i < 21; i++) {
+ answersThree.add(i);
+ }
+ Assert.assertEquals(answersThree, processResult(resultSet));
+
+ valueCounter.teardown();
+ }
+
+ public static void insertValues()
+ {
+ logger.debug("start round 0");
+ valueCounter.beginWindow(0);
+ emitKeyVals(1, 1, 10, 1);
+ emitKeyVals(1, 5, 15, 1);
+ valueCounter.endWindow();
+ logger.debug("end round 0");
+
+ logger.debug("start round 1");
+ valueCounter.beginWindow(1);
+ emitKeyVals(2, 3, 15, 3);
+ emitKeyVals(3, 2, 20, 2);
+ emitKeyVals(3, 11, 20, 1);
+ valueCounter.endWindow();
+ logger.debug("end round 1");
+
+ logger.debug("start round 2");
+ valueCounter.beginWindow(2);
+ emitKeyVals(3, 2, 20, 2);
+ emitKeyVals(2, 11, 20, 1);
+ valueCounter.endWindow();
+ logger.debug("end round 2");
+ }
+
+ public static ArrayList<Integer> processResult(ResultSet resultSet)
+ {
+ ArrayList<Integer> tempList = new ArrayList<Integer>();
+ try {
+ while (resultSet.next()) {
+ tempList.add(resultSet.getInt(1));
+ }
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+ Collections.sort(tempList);
+ return tempList;
+ }
+
+ public static void emitKeyVals(int key, int start, int end, int increment)
+ {
+ int count = 0;
+ Set<Object> valSet = new HashSet<Object>();
+ for (int i = start; i <= end; i += increment) {
+ count++;
+ valSet.add(i);
+ }
+ valueCounter.processTuple(new InternalCountOutput<Integer>(key, count, valSet));
+ }
+
+ @Test
+ public void testSetup() throws Exception
+ {
+ insertValues();
+ Statement stmt = valueCounter.getStore().getConnection().createStatement();
+ com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap attributes = new com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap();
+ attributes.put(DAG.APPLICATION_ID, APP_ID);
+ attributes.put(DAG.APPLICATION_PATH, applicationPath);
+ attributes.put(OperatorContext.ACTIVATION_WINDOW_ID, 2L);
+
+ valueCounter.setup(new OperatorContextTestHelper.TestIdOperatorContext(0, attributes));
+
+ ResultSet resultSet = stmt.executeQuery("SELECT col2 FROM " + TABLE_NAME + " WHERE col1 = 2");
+ ArrayList<Integer> answersAfterClear = new ArrayList<Integer>();
+ for (int i = 3; i < 16; i += 3) {
+ answersAfterClear.add(i);
+ }
+ Assert.assertEquals(answersAfterClear, processResult(resultSet));
+
+ resultSet = stmt.executeQuery("SELECT col2 FROM " + TABLE_NAME + " WHERE col1 = 3");
+ ArrayList<Integer> answersThree = new ArrayList<Integer>();
+ answersThree.add(2);
+ answersThree.add(4);
+ answersThree.add(6);
+ answersThree.add(8);
+ answersThree.add(10);
+ for (int i = 11; i < 21; i++) {
+ answersThree.add(i);
+ }
+ Assert.assertEquals(answersThree, processResult(resultSet));
+ stmt.executeQuery("DELETE FROM " + TABLE_NAME);
+ }
+
+ @BeforeClass
+ public static void setup() throws Exception
+ {
+ valueCounter = new IntegerUniqueValueCountAppender();
+ Class.forName(INMEM_DB_DRIVER).newInstance();
+ Connection con = DriverManager.getConnection(INMEM_DB_URL, new Properties());
+ Statement stmt = con.createStatement();
+ stmt.execute("CREATE TABLE IF NOT EXISTS " + TABLE_NAME + " (col1 INTEGER, col2 INTEGER, col3 BIGINT)");
+ com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap attributes = new com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap();
+ attributes.put(DAG.APPLICATION_ID, APP_ID);
+ attributes.put(DAG.APPLICATION_PATH, applicationPath);
+ attributes.put(OperatorContext.ACTIVATION_WINDOW_ID, 0L);
+ valueCounter.setTableName(TABLE_NAME);
+ valueCounter.getStore().setDatabaseDriver(INMEM_DB_DRIVER);
+ valueCounter.getStore().setDatabaseUrl(INMEM_DB_URL);
+ TestIdOperatorContext context = new OperatorContextTestHelper.TestIdOperatorContext(OPERATOR_ID, attributes);
+ valueCounter.setup(context);
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/distributedistinct/src/test/java/org/apache/apex/examples/distributeddistinct/StatefulApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/distributedistinct/src/test/java/org/apache/apex/examples/distributeddistinct/StatefulApplicationTest.java b/examples/distributedistinct/src/test/java/org/apache/apex/examples/distributeddistinct/StatefulApplicationTest.java
new file mode 100644
index 0000000..5e50f6c
--- /dev/null
+++ b/examples/distributedistinct/src/test/java/org/apache/apex/examples/distributeddistinct/StatefulApplicationTest.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.distributeddistinct;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Properties;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.LocalMode;
+
+public class StatefulApplicationTest
+{
+
+ @BeforeClass
+ public static void setup()
+ {
+ try {
+ Class.forName(StatefulUniqueCountTest.INMEM_DB_DRIVER).newInstance();
+ Connection con = DriverManager.getConnection(StatefulUniqueCountTest.INMEM_DB_URL, new Properties());
+ Statement stmt = con.createStatement();
+ stmt.execute("CREATE TABLE IF NOT EXISTS " + StatefulUniqueCountTest.TABLE_NAME + " (col1 INTEGER, col2 INTEGER, col3 BIGINT)");
+ } catch (InstantiationException e) {
+ throw new RuntimeException(e);
+ } catch (IllegalAccessException e) {
+ throw new RuntimeException(e);
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException(e);
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Test
+ public void testApplication() throws Exception
+ {
+ LocalMode lma = LocalMode.newInstance();
+ Configuration conf = new Configuration(false);
+ conf.set("dt.operator.StatefulUniqueCounter.prop.tableName", "Test_Lookup_Cache");
+ conf.set("dt.operator.StatefulUniqueCounter.prop.store.dbUrl", "jdbc:hsqldb:mem:test;sql.syntax_mys=true");
+ conf.set("dt.operator.StatefulUniqueCounter.prop.store.dbDriver", "org.hsqldb.jdbcDriver");
+
+ lma.prepareDAG(new StatefulApplication(), conf);
+ lma.cloneDAG();
+ LocalMode.Controller lc = lma.getController();
+ lc.setHeartbeatMonitoringEnabled(false);
+ lc.runAsync();
+
+ long now = System.currentTimeMillis();
+ while (System.currentTimeMillis() - now < 15000) {
+ Thread.sleep(1000);
+ }
+
+ lc.shutdown();
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/distributedistinct/src/test/java/org/apache/apex/examples/distributeddistinct/StatefulUniqueCountTest.java
----------------------------------------------------------------------
diff --git a/examples/distributedistinct/src/test/java/org/apache/apex/examples/distributeddistinct/StatefulUniqueCountTest.java b/examples/distributedistinct/src/test/java/org/apache/apex/examples/distributeddistinct/StatefulUniqueCountTest.java
new file mode 100644
index 0000000..6821431
--- /dev/null
+++ b/examples/distributedistinct/src/test/java/org/apache/apex/examples/distributeddistinct/StatefulUniqueCountTest.java
@@ -0,0 +1,248 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.distributeddistinct;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Properties;
+
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.Context.OperatorContext;
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.DefaultInputPort;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.InputOperator;
+import com.datatorrent.api.LocalMode;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.common.util.BaseOperator;
+
+import com.datatorrent.lib.algo.UniqueValueCount;
+import com.datatorrent.lib.util.KeyValPair;
+
+public class StatefulUniqueCountTest
+{
+
+ public static final String INMEM_DB_URL = "jdbc:hsqldb:mem:test;sql.syntax_mys=true";
+ public static final String INMEM_DB_DRIVER = "org.hsqldb.jdbc.JDBCDriver";
+ public static final String TABLE_NAME = "Test_Lookup_Cache";
+
+ static class KeyGen implements InputOperator
+ {
+
+ public transient DefaultOutputPort<KeyValPair<Integer, Object>> output = new DefaultOutputPort<KeyValPair<Integer, Object>>();
+
+ @Override
+ public void beginWindow(long windowId)
+ {
+ }
+
+ public void emitKeyVals(int key, int start, int end, int increment)
+ {
+ for (int i = start; i <= end; i += increment) {
+ output.emit(new KeyValPair<Integer, Object>(key, i));
+ }
+ }
+
+ @Override
+ public void endWindow()
+ {
+ }
+
+ @Override
+ public void setup(OperatorContext context)
+ {
+
+ }
+
+ @Override
+ public void teardown()
+ {
+
+ }
+
+ @Override
+ public void emitTuples()
+ {
+ emitKeyVals(1, 1, 10, 1);
+ emitKeyVals(2, 3, 15, 3);
+ emitKeyVals(3, 2, 20, 2);
+ emitKeyVals(1, 5, 15, 1);
+ emitKeyVals(2, 11, 20, 1);
+ emitKeyVals(3, 11, 20, 1);
+ }
+ }
+
+ static class VerifyTable extends BaseOperator
+ {
+
+ private static final String INMEM_DB_URL = "jdbc:hsqldb:mem:test;sql.syntax_mys=true";
+ private static final String INMEM_DB_DRIVER = "org.hsqldb.jdbc.JDBCDriver";
+ protected static final String TABLE_NAME = "Test_Lookup_Cache";
+
+ public final transient DefaultInputPort<Object> input = new DefaultInputPort<Object>()
+ {
+ @Override
+ public void process(Object tuple)
+ {
+ }
+ };
+
+ @Override
+ public void beginWindow(long windowId)
+ {
+ }
+
+ @Override
+ public void endWindow()
+ {
+ try {
+ Class.forName(INMEM_DB_DRIVER).newInstance();
+ Connection con = DriverManager.getConnection(INMEM_DB_URL, new Properties());
+ Statement stmt = con.createStatement();
+ ResultSet resultSet = stmt.executeQuery("SELECT col2 FROM " + TABLE_NAME + " WHERE col1 = 1");
+ ArrayList<Integer> answersOne = new ArrayList<Integer>();
+ for (int i = 1; i < 16; i++) {
+ answersOne.add(i);
+ }
+ Assert.assertEquals(answersOne, processResult(resultSet));
+
+ resultSet = stmt.executeQuery("SELECT col2 FROM " + TABLE_NAME + " WHERE col1 = 2");
+ ArrayList<Integer> answersTwo = new ArrayList<Integer>();
+ answersTwo.add(3);
+ answersTwo.add(6);
+ answersTwo.add(9);
+ for (int i = 11; i < 21; i++) {
+ answersTwo.add(i);
+ }
+ Assert.assertEquals(answersTwo, processResult(resultSet));
+
+ resultSet = stmt.executeQuery("SELECT col2 FROM " + TABLE_NAME + " WHERE col1 = 3");
+ ArrayList<Integer> answersThree = new ArrayList<Integer>();
+ answersThree.add(2);
+ answersThree.add(4);
+ answersThree.add(6);
+ answersThree.add(8);
+ answersThree.add(10);
+ for (int i = 11; i < 21; i++) {
+ answersThree.add(i);
+ }
+ Assert.assertEquals(answersThree, processResult(resultSet));
+ } catch (Throwable e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void setup(OperatorContext context)
+ {
+ }
+
+ @Override
+ public void teardown()
+ {
+
+ }
+
+ public static ArrayList<Integer> processResult(ResultSet resultSet)
+ {
+ ArrayList<Integer> tempList = new ArrayList<Integer>();
+ try {
+ while (resultSet.next()) {
+ tempList.add(resultSet.getInt(1));
+ }
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+ Collections.sort(tempList);
+ return tempList;
+ }
+ }
+
+ public class Application implements StreamingApplication
+ {
+ @SuppressWarnings("unchecked")
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ KeyGen keyGen = dag.addOperator("KeyGenerator", new KeyGen());
+ UniqueValueCount<Integer> valCount = dag.addOperator("ValueCounter", new UniqueValueCount<Integer>());
+ IntegerUniqueValueCountAppender uniqueUnifier = dag.addOperator("Unique", new IntegerUniqueValueCountAppender());
+ VerifyTable verifyTable = dag.addOperator("VerifyTable", new VerifyTable());
+
+ @SuppressWarnings("rawtypes")
+ DefaultOutputPort valOut = valCount.output;
+ @SuppressWarnings("rawtypes")
+ DefaultOutputPort uniqueOut = uniqueUnifier.output;
+ dag.addStream("DataIn", keyGen.output, valCount.input);
+ dag.addStream("UnifyWindows", valOut, uniqueUnifier.input);
+ dag.addStream("ResultsOut", uniqueOut, verifyTable.input);
+ }
+ }
+
+ @BeforeClass
+ public static void setup()
+ {
+ try {
+ Class.forName(INMEM_DB_DRIVER).newInstance();
+ Connection con = DriverManager.getConnection(INMEM_DB_URL, new Properties());
+ Statement stmt = con.createStatement();
+ stmt.execute("CREATE TABLE IF NOT EXISTS " + TABLE_NAME + " (col1 INTEGER, col2 INTEGER, col3 BIGINT)");
+ } catch (InstantiationException e) {
+ throw new RuntimeException(e);
+ } catch (IllegalAccessException e) {
+ throw new RuntimeException(e);
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException(e);
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Test
+ public void testApplication() throws Exception
+ {
+ LocalMode lma = LocalMode.newInstance();
+ Configuration conf = new Configuration(false);
+ conf.set("dt.operator.Unique.prop.tableName", "Test_Lookup_Cache");
+ conf.set("dt.operator.Unique.prop.store.dbUrl", "jdbc:hsqldb:mem:test;sql.syntax_mys=true");
+ conf.set("dt.operator.Unique.prop.store.dbDriver", "org.hsqldb.jdbcDriver");
+
+ lma.prepareDAG(new Application(), conf);
+ lma.cloneDAG();
+ LocalMode.Controller lc = lma.getController();
+ lc.setHeartbeatMonitoringEnabled(false);
+ lc.runAsync();
+
+ long now = System.currentTimeMillis();
+ while (System.currentTimeMillis() - now < 15000) {
+ Thread.sleep(1000);
+ }
+ lc.shutdown();
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/distributedistinct/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/examples/distributedistinct/src/test/resources/log4j.properties b/examples/distributedistinct/src/test/resources/log4j.properties
new file mode 100644
index 0000000..cf0d19e
--- /dev/null
+++ b/examples/distributedistinct/src/test/resources/log4j.properties
@@ -0,0 +1,43 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+log4j.rootLogger=DEBUG,CONSOLE
+
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+log4j.appender.CONSOLE.threshold=${test.log.console.threshold}
+test.log.console.threshold=DEBUG
+
+log4j.appender.RFA=org.apache.log4j.RollingFileAppender
+log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
+log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+log4j.appender.RFA.File=/tmp/app.log
+
+# to enable, add SYSLOG to rootLogger
+log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender
+log4j.appender.SYSLOG.syslogHost=127.0.0.1
+log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout
+log4j.appender.SYSLOG.layout.conversionPattern=${dt.cid} %-5p [%t] %c{2} %x - %m%n
+log4j.appender.SYSLOG.Facility=LOCAL1
+
+log4j.logger.org=info
+#log4j.logger.org.apache.commons.beanutils=warn
+log4j.logger.com.datatorrent=debug
+log4j.logger.org.apache.apex=debug
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/echoserver/pom.xml
----------------------------------------------------------------------
diff --git a/examples/echoserver/pom.xml b/examples/echoserver/pom.xml
new file mode 100644
index 0000000..ae8a4be
--- /dev/null
+++ b/examples/echoserver/pom.xml
@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>malhar-examples</artifactId>
+ <groupId>org.apache.apex</groupId>
+ <version>3.7.0-SNAPSHOT</version>
+ </parent>
+
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-examples-echoserver</artifactId>
+ <packaging>jar</packaging>
+
+ <!-- change these to the appropriate values -->
+ <name>Apache Apex Malhar EchoServer Example</name>
+ <description>An example server that echos data sent by a network client back to it</description>
+</project>
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/echoserver/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/examples/echoserver/src/assemble/appPackage.xml b/examples/echoserver/src/assemble/appPackage.xml
new file mode 100644
index 0000000..4138cf2
--- /dev/null
+++ b/examples/echoserver/src/assemble/appPackage.xml
@@ -0,0 +1,59 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+ <id>appPackage</id>
+ <formats>
+ <format>jar</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>${basedir}/target/</directory>
+ <outputDirectory>/app</outputDirectory>
+ <includes>
+ <include>${project.artifactId}-${project.version}.jar</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/target/deps</directory>
+ <outputDirectory>/lib</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/site/conf</directory>
+ <outputDirectory>/conf</outputDirectory>
+ <includes>
+ <include>*.xml</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/META-INF</directory>
+ <outputDirectory>/META-INF</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/app</directory>
+ <outputDirectory>/app</outputDirectory>
+ </fileSet>
+ </fileSets>
+
+</assembly>
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/echoserver/src/main/java/org/apache/apex/examples/echoserver/Application.java
----------------------------------------------------------------------
diff --git a/examples/echoserver/src/main/java/org/apache/apex/examples/echoserver/Application.java b/examples/echoserver/src/main/java/org/apache/apex/examples/echoserver/Application.java
new file mode 100644
index 0000000..e59f160
--- /dev/null
+++ b/examples/echoserver/src/main/java/org/apache/apex/examples/echoserver/Application.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.echoserver;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+
+/**
+ * @since 2.1.0
+ */
+@ApplicationAnnotation(name = "EchoServer")
+public class Application implements StreamingApplication
+{
+
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ MessageReceiver receiver = dag.addOperator("Message Receiver", MessageReceiver.class);
+ MessageResponder responder = dag.addOperator("Message Responder", MessageResponder.class);
+ // Locality has to be container so that the operators use the same socket
+ dag.addStream("messages", receiver.messageOutput, responder.messageInput).setLocality(DAG.Locality.CONTAINER_LOCAL);
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/echoserver/src/main/java/org/apache/apex/examples/echoserver/Message.java
----------------------------------------------------------------------
diff --git a/examples/echoserver/src/main/java/org/apache/apex/examples/echoserver/Message.java b/examples/echoserver/src/main/java/org/apache/apex/examples/echoserver/Message.java
new file mode 100644
index 0000000..3790ea5
--- /dev/null
+++ b/examples/echoserver/src/main/java/org/apache/apex/examples/echoserver/Message.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.echoserver;
+
+import java.net.SocketAddress;
+
+/**
+ * @since 2.1.0
+ */
+public class Message
+{
+ public String message;
+ public SocketAddress socketAddress;
+}
[10/30] apex-malhar git commit: Renamed demos to examples. Packages
and artifactid names are changed as suggested.
Posted by th...@apache.org.
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/iteration/pom.xml
----------------------------------------------------------------------
diff --git a/examples/iteration/pom.xml b/examples/iteration/pom.xml
new file mode 100644
index 0000000..471b173
--- /dev/null
+++ b/examples/iteration/pom.xml
@@ -0,0 +1,37 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>malhar-examples-iteration</artifactId>
+ <packaging>jar</packaging>
+
+ <name>Apache Apex Malhar Iteration Example</name>
+ <description>DataTorrent example applications that demonstrates the iteration feature.</description>
+
+ <parent>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-examples</artifactId>
+ <version>3.7.0-SNAPSHOT</version>
+ </parent>
+
+</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/iteration/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/examples/iteration/src/assemble/appPackage.xml b/examples/iteration/src/assemble/appPackage.xml
new file mode 100644
index 0000000..4138cf2
--- /dev/null
+++ b/examples/iteration/src/assemble/appPackage.xml
@@ -0,0 +1,59 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+ <id>appPackage</id>
+ <formats>
+ <format>jar</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>${basedir}/target/</directory>
+ <outputDirectory>/app</outputDirectory>
+ <includes>
+ <include>${project.artifactId}-${project.version}.jar</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/target/deps</directory>
+ <outputDirectory>/lib</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/site/conf</directory>
+ <outputDirectory>/conf</outputDirectory>
+ <includes>
+ <include>*.xml</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/META-INF</directory>
+ <outputDirectory>/META-INF</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/app</directory>
+ <outputDirectory>/app</outputDirectory>
+ </fileSet>
+ </fileSets>
+
+</assembly>
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/iteration/src/main/java/org/apache/apex/examples/iteration/Application.java
----------------------------------------------------------------------
diff --git a/examples/iteration/src/main/java/org/apache/apex/examples/iteration/Application.java b/examples/iteration/src/main/java/org/apache/apex/examples/iteration/Application.java
new file mode 100644
index 0000000..4b7bae9
--- /dev/null
+++ b/examples/iteration/src/main/java/org/apache/apex/examples/iteration/Application.java
@@ -0,0 +1,171 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.iteration;
+
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.Context;
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.DefaultInputPort;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.common.util.BaseOperator;
+import com.datatorrent.common.util.DefaultDelayOperator;
+import com.datatorrent.lib.testbench.RandomEventGenerator;
+
+/**
+ * Iteration example : <br>
+ *
+ * <pre>
+ * LocalMode.runApp(new Application(), 600000); // 10 min run
+ * </pre>
+ *
+ * Run Success : <br>
+ * For successful deployment and run, user should see the Fibonacci sequence, something like the
+ * following output on the console:
+ *
+ * <pre>
+ * 1
+ * 1
+ * 2
+ * 3
+ * 5
+ * 8
+ * 13
+ * 21
+ * 34
+ * 55
+ * ...
+ * </pre>
+ *
+ *
+ * @since 3.4.0
+ */
+@ApplicationAnnotation(name = "IterationExample")
+public class Application implements StreamingApplication
+{
+ private static final Logger LOG = LoggerFactory.getLogger(Application.class);
+ private String extraOutputFileName; // for unit test
+
+ public static class FibonacciOperator extends BaseOperator
+ {
+ public long currentNumber = 1;
+ private transient long tempNum;
+ public transient DefaultInputPort<Object> dummyInputPort = new DefaultInputPort<Object>()
+ {
+ @Override
+ public void process(Object tuple)
+ {
+ }
+ };
+ public transient DefaultInputPort<Long> input = new DefaultInputPort<Long>()
+ {
+ @Override
+ public void process(Long tuple)
+ {
+ tempNum = (currentNumber == 1) ? 1 : tuple;
+ }
+ };
+ public transient DefaultOutputPort<Long> output = new DefaultOutputPort<>();
+
+
+ @Override
+ public void endWindow()
+ {
+ output.emit(currentNumber);
+ currentNumber += tempNum;
+ if (currentNumber <= 0) {
+ currentNumber = 1;
+ }
+ }
+ }
+
+ public static class StdoutOperator extends BaseOperator
+ {
+ private String extraOutputFileName; // for unit test
+ private transient PrintStream extraOutputStream;
+ /**
+ * This is the input port which receives the tuples that will be written to stdout.
+ */
+ public final transient DefaultInputPort<Object> input = new DefaultInputPort<Object>()
+ {
+ @Override
+ @SuppressWarnings("UseOfSystemOutOrSystemErr")
+ public void process(Object t)
+ {
+ String s = t.toString();
+ LOG.info(s);
+ if (extraOutputStream != null) {
+ extraOutputStream.println(s);
+ }
+ }
+ };
+
+ @Override
+ public void setup(Context.OperatorContext context)
+ {
+ if (extraOutputFileName != null) {
+ try {
+ extraOutputStream = new PrintStream(new FileOutputStream(extraOutputFileName), true);
+ } catch (IOException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+ }
+
+ @Override
+ public void teardown()
+ {
+ extraOutputStream.close();
+ }
+
+ public void setExtraOutputFileName(String fileName)
+ {
+ this.extraOutputFileName = fileName;
+ }
+ }
+
+ public void setExtraOutputFileName(String fileName)
+ {
+ this.extraOutputFileName = fileName;
+ }
+
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ RandomEventGenerator rand = dag.addOperator("rand", new RandomEventGenerator());
+ FibonacciOperator fib = dag.addOperator("FIB", FibonacciOperator.class);
+ DefaultDelayOperator opDelay = dag.addOperator("opDelay", DefaultDelayOperator.class);
+ StdoutOperator console = new StdoutOperator();
+ console.setExtraOutputFileName(extraOutputFileName);
+ dag.addOperator("console", console);
+ dag.addStream("dummy_to_operator", rand.integer_data, fib.dummyInputPort);
+ dag.addStream("operator_to_delay", fib.output, opDelay.input, console.input);
+ dag.addStream("delay_to_operator", opDelay.output, fib.input);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/iteration/src/main/java/org/apache/apex/examples/iteration/package-info.java
----------------------------------------------------------------------
diff --git a/examples/iteration/src/main/java/org/apache/apex/examples/iteration/package-info.java b/examples/iteration/src/main/java/org/apache/apex/examples/iteration/package-info.java
new file mode 100644
index 0000000..e87da8b
--- /dev/null
+++ b/examples/iteration/src/main/java/org/apache/apex/examples/iteration/package-info.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/**
+ * Iteration demonstration application.
+ */
+package org.apache.apex.examples.iteration;
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/iteration/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/examples/iteration/src/main/resources/META-INF/properties.xml b/examples/iteration/src/main/resources/META-INF/properties.xml
new file mode 100644
index 0000000..6b88913
--- /dev/null
+++ b/examples/iteration/src/main/resources/META-INF/properties.xml
@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<configuration>
+ <!-- Memory settings for all examples -->
+ <property>
+ <name>dt.attr.MASTER_MEMORY_MB</name>
+ <value>512</value>
+ </property>
+ <property>
+ <name>dt.attr.DEBUG</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>dt.application.*.operator.*.attr.MEMORY_MB</name>
+ <value>128</value>
+ </property>
+ <property>
+ <name>dt.application.*.operator.*.attr.JVM_OPTIONS</name>
+ <value>-Xmx128M</value>
+ </property>
+ <property>
+ <name>dt.application.*.operator.*.port.*.attr.BUFFER_MEMORY_MB</name>
+ <value>128</value>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/iteration/src/test/java/org/apache/apex/examples/iteration/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/iteration/src/test/java/org/apache/apex/examples/iteration/ApplicationTest.java b/examples/iteration/src/test/java/org/apache/apex/examples/iteration/ApplicationTest.java
new file mode 100644
index 0000000..1a6c5e0
--- /dev/null
+++ b/examples/iteration/src/test/java/org/apache/apex/examples/iteration/ApplicationTest.java
@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.iteration;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.LocalMode;
+
+/**
+ *
+ */
+public class ApplicationTest
+{
+ @Test
+ public void testIterationApp() throws Exception
+ {
+ LocalMode lma = LocalMode.newInstance();
+ Configuration conf = new Configuration(false);
+ Application app = new Application();
+ String outputFileName = "target/output.txt";
+ long timeout = 10 * 1000; // 10 seconds
+
+ new File(outputFileName).delete();
+ app.setExtraOutputFileName(outputFileName);
+ lma.prepareDAG(app, conf);
+ LocalMode.Controller lc = lma.getController();
+ lc.runAsync();
+
+ long startTime = System.currentTimeMillis();
+ do {
+ try {
+ Thread.sleep(500);
+ } catch (InterruptedException ex) {
+ break;
+ }
+ File file = new File(outputFileName);
+ if (file.length() > 50) {
+ break;
+ }
+ }
+ while (System.currentTimeMillis() - startTime < timeout);
+
+ lc.shutdown();
+ try (BufferedReader br = new BufferedReader(new FileReader(outputFileName))) {
+ Assert.assertEquals("1", br.readLine());
+ Assert.assertEquals("1", br.readLine());
+ Assert.assertEquals("2", br.readLine());
+ Assert.assertEquals("3", br.readLine());
+ Assert.assertEquals("5", br.readLine());
+ Assert.assertEquals("8", br.readLine());
+ Assert.assertEquals("13", br.readLine());
+ Assert.assertEquals("21", br.readLine());
+ Assert.assertEquals("34", br.readLine());
+ Assert.assertEquals("55", br.readLine());
+ Assert.assertEquals("89", br.readLine());
+ Assert.assertEquals("144", br.readLine());
+ Assert.assertEquals("233", br.readLine());
+ Assert.assertEquals("377", br.readLine());
+ Assert.assertEquals("610", br.readLine());
+ Assert.assertEquals("987", br.readLine());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/iteration/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/examples/iteration/src/test/resources/log4j.properties b/examples/iteration/src/test/resources/log4j.properties
new file mode 100644
index 0000000..cf0d19e
--- /dev/null
+++ b/examples/iteration/src/test/resources/log4j.properties
@@ -0,0 +1,43 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+log4j.rootLogger=DEBUG,CONSOLE
+
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+log4j.appender.CONSOLE.threshold=${test.log.console.threshold}
+test.log.console.threshold=DEBUG
+
+log4j.appender.RFA=org.apache.log4j.RollingFileAppender
+log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
+log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+log4j.appender.RFA.File=/tmp/app.log
+
+# to enable, add SYSLOG to rootLogger
+log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender
+log4j.appender.SYSLOG.syslogHost=127.0.0.1
+log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout
+log4j.appender.SYSLOG.layout.conversionPattern=${dt.cid} %-5p [%t] %c{2} %x - %m%n
+log4j.appender.SYSLOG.Facility=LOCAL1
+
+log4j.logger.org=info
+#log4j.logger.org.apache.commons.beanutils=warn
+log4j.logger.com.datatorrent=debug
+log4j.logger.org.apache.apex=debug
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/machinedata/pom.xml
----------------------------------------------------------------------
diff --git a/examples/machinedata/pom.xml b/examples/machinedata/pom.xml
new file mode 100644
index 0000000..cf17d9b
--- /dev/null
+++ b/examples/machinedata/pom.xml
@@ -0,0 +1,66 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>malhar-examples-machinedata</artifactId>
+ <packaging>jar</packaging>
+
+ <name>Apache Apex Malhar MachineData Example</name>
+ <description></description>
+
+ <parent>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-examples</artifactId>
+ <version>3.7.0-SNAPSHOT</version>
+ </parent>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-contrib</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>redis.clients</groupId>
+ <artifactId>jedis</artifactId>
+ <version>2.5.1</version>
+ </dependency>
+ <dependency>
+ <groupId>javax.mail</groupId>
+ <artifactId>javax.mail-api</artifactId>
+ <version>1.5.2</version>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.mail</groupId>
+ <artifactId>javax.mail</artifactId>
+ <version>1.5.2</version>
+ </dependency>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/machinedata/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/examples/machinedata/src/assemble/appPackage.xml b/examples/machinedata/src/assemble/appPackage.xml
new file mode 100644
index 0000000..4138cf2
--- /dev/null
+++ b/examples/machinedata/src/assemble/appPackage.xml
@@ -0,0 +1,59 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+ <id>appPackage</id>
+ <formats>
+ <format>jar</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>${basedir}/target/</directory>
+ <outputDirectory>/app</outputDirectory>
+ <includes>
+ <include>${project.artifactId}-${project.version}.jar</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/target/deps</directory>
+ <outputDirectory>/lib</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/site/conf</directory>
+ <outputDirectory>/conf</outputDirectory>
+ <includes>
+ <include>*.xml</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/META-INF</directory>
+ <outputDirectory>/META-INF</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/app</directory>
+ <outputDirectory>/app</outputDirectory>
+ </fileSet>
+ </fileSets>
+
+</assembly>
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/machinedata/src/main/html/global.js
----------------------------------------------------------------------
diff --git a/examples/machinedata/src/main/html/global.js b/examples/machinedata/src/main/html/global.js
new file mode 100644
index 0000000..753f58f
--- /dev/null
+++ b/examples/machinedata/src/main/html/global.js
@@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/**
+ * Declaration and initialization for global variables.
+ */
+
+// url parameters
+var params;
+
+// Data Points
+var aggrData;
+var aggrDataPoints;
+var contData;
+var contDataPoints;
+
+// CPU data table
+var cpuTable;
+var cpuChart;
+var cpuView;
+
+// ram data table
+var ramTable;
+var ramChart;
+var ramView;
+
+// hdd data table
+var hddTable;
+var hddChart;
+var hddView;
+
+// chart options
+var chartOptions;
+
+// Date formatter
+var dateFormatter;
+
+// window look back value
+var lookback;
+var aggrLookBack;
+var contLookBack;
+var contRefresh;
+
+// Get split query string
+function QueryString() {
+ var query_string = {};
+ var query = window.location.search.substring(1);
+ return query;
+}
+function SplitQuery(query)
+{
+ var params = {};
+ var vars = query.split("&");
+ for (var i=0;i<vars.length;i++)
+ {
+ var pair = vars[i].split("=");
+ if(pair.length == 2)
+ {
+ params[pair[0]] = pair[1];
+ }
+ }
+ return params;
+}
+
+// Initialize global variable(s)
+function InitializeGlobal()
+{
+ // Initialize params
+ params = SplitQuery(QueryString());
+
+ // Initialize data points
+ aggrDataPoints = new Array();
+ contDataPoints = new Array();
+
+ // Initialize cpu table
+ cpuTable = new google.visualization.DataTable();
+ cpuTable.addColumn('datetime', 'Time');
+ cpuTable.addColumn('number', 'CPU');
+ chartOptions = { width: 600, height: 300, legend: 'none', pointSize: 0, lineWidth : 1 };
+ cpuChart = new google.visualization.ScatterChart(document.getElementById('chart_div'));
+ cpuView = new google.visualization.DataView(cpuTable);
+
+ // Initialize ram table
+ ramTable = new google.visualization.DataTable();
+ ramTable.addColumn('datetime', 'Time');
+ ramTable.addColumn('number', 'RAM');;
+ ramChart = new google.visualization.ScatterChart(document.getElementById('chart1_div'));
+ ramView = new google.visualization.DataView(ramTable);
+
+ // Initialize hdd table
+ hddTable = new google.visualization.DataTable();
+ hddTable.addColumn('datetime', 'Time');
+ hddTable.addColumn('number', 'HDD');;
+ hddChart = new google.visualization.ScatterChart(document.getElementById('chart2_div'));
+ hddView = new google.visualization.DataView(hddTable);
+
+ // get lookback value
+ lookback = (new Date().getTime()/1000) - 3600*6;
+ if (params['lookback'] && (params['lookback'].length > 0)) lookback = (new Date().getTime()/1000) - (3600*(parseInt(params['lookback'])));
+ aggrLookBack = lookback;
+
+ // get continuos lookback
+ contLookBack = lookback;
+ contRefresh = 5;
+
+ // get param lookback
+ paramLookBack = 6;
+ if (params['lookback'] && (params['lookback'].length > 0)) paramLookBack = parseInt(params['lookback']);
+ //if (params['refresh'] && (params['refresh'].length > 0)) contRefresh = parseInt(params['refresh']);
+}
+
+
+/**
+ * Function to create fetch urls from given parameters
+ */
+function DataUrl()
+{
+ var url = "json.php?bucket=m";
+ url += "&customer=";
+ if (params['customer'])
+ {
+ url += params['customer'];
+ }
+ url += "&product=";
+ if (params['product'])
+ {
+ url += params['product'];
+ }
+ url += "&os=";
+ if (params['os'])
+ {
+ url += params['os'];
+ }
+ url += "&software1=";
+ if (params['software1'])
+ {
+ url += params['software1'];
+ }
+ url += "&software2=";
+ if (params['software2'])
+ {
+ url += params['software2'];
+ }
+ url += "&software3=";
+ if (params['software3'])
+ {
+ url += params['software3'];
+ }
+ url += "&from=";
+ url += Math.floor(lookback);
+ return url;
+}
+
+/**
+ * Creates data table with time stamp and cpu values.
+ * Draw line chart for time vs cpu.
+ */
+function DrawCPUChart()
+{
+ // create/delete rows
+ if (cpuTable.getNumberOfRows() < aggrDataPoints.length)
+ {
+ var numRows = aggrDataPoints.length - cpuTable.getNumberOfRows();
+ cpuTable.addRows(numRows);
+ } else {
+ for(var i=(cpuTable.getNumberOfRows()-1); i >= aggrDataPoints.length; i--)
+ {
+ cpuTable.removeRow(i);
+ }
+ }
+
+ // Populate data table with time/cpu data points.
+ for(var i=0; i < cpuTable.getNumberOfRows(); i++)
+ {
+ //if(parseFloat(aggrDataPoints[i].cpu) < 500) continue;
+ cpuTable.setCell(i, 0, new Date(parseInt(aggrDataPoints[i].timestamp)));
+ cpuTable.setCell(i, 1, parseFloat(aggrDataPoints[i].cpu));
+ }
+
+ // Draw line chart.
+ chartOptions.title = 'CPU Usage (%)';
+ cpuChart.draw(cpuView, chartOptions);
+}
+
+/**
+ * Creates data table with time stamp and revenu values.
+ * Draw line chart for time vs ram.
+ */
+function DrawRAMChart()
+{
+ // create/delete rows
+ if (ramTable.getNumberOfRows() < aggrDataPoints.length)
+ {
+ var numRows = aggrDataPoints.length - ramTable.getNumberOfRows();
+ ramTable.addRows(numRows);
+ } else {
+ for(var i=(ramTable.getNumberOfRows()-1); i >= aggrDataPoints.length; i--)
+ {
+ ramTable.removeRow(i);
+ }
+ }
+
+ // Populate data table with time/ram data points.
+ for(var i=0; i < ramTable.getNumberOfRows(); i++)
+ {
+ ramTable.setCell(i, 0, new Date(parseInt(aggrDataPoints[i].timestamp)));
+ ramTable.setCell(i, 1, parseFloat(aggrDataPoints[i].ram));
+ }
+
+ // Draw line chart.
+ chartOptions.title = 'RAM Usage (%)';
+ ramChart.draw(ramView, chartOptions);
+}
+
+/**
+ * Creates data table with time stamp and hdd values.
+ * Draw line chart for time vs hdd.
+ */
+function DrawHDDChart()
+{
+ // create/delete rows
+ if (hddTable.getNumberOfRows() < aggrDataPoints.length)
+ {
+ var numRows = aggrDataPoints.length - hddTable.getNumberOfRows();
+ hddTable.addRows(numRows);
+ } else {
+ for(var i=(hddTable.getNumberOfRows()-1); i >= aggrDataPoints.length; i--)
+ {
+ hddTable.removeRow(i);
+ }
+ }
+
+ // Populate data table with time/hdd data points.
+ for(var i=0; i < hddTable.getNumberOfRows(); i++)
+ {
+ hddTable.setCell(i, 0, new Date(parseInt(aggrDataPoints[i].timestamp)));
+ hddTable.setCell(i, 1, parseInt(aggrDataPoints[i].hdd));
+ }
+
+ // Draw line chart.
+ chartOptions.title = 'HDD Usage (%)';
+ hddChart.draw(hddView, chartOptions);
+}
+
+/**
+ * Sort json array
+ */
+function sortByKey(array, key) {
+ return array.sort(function(a, b) {
+ var x = a[key]; var y = b[key];
+ return ((x < y) ? -1 : ((x > y) ? 1 : 0));
+ });
+}
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/machinedata/src/main/html/index.php
----------------------------------------------------------------------
diff --git a/examples/machinedata/src/main/html/index.php b/examples/machinedata/src/main/html/index.php
new file mode 100644
index 0000000..d52a960
--- /dev/null
+++ b/examples/machinedata/src/main/html/index.php
@@ -0,0 +1,263 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
+<html>
+<head>
+<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>Machine Generated Data Example </title>
+
+<link rel="stylesheet" type="text/css" href="malhar.css">
+
+<!-- Google charts include -->
+<script type="text/javascript" src="https://www.google.com/jsapi"></script>
+<script type="text/javascript">
+google.load('visualization', '1', {'packages':['corechart']});
+</script>
+
+<!-- Malhar charting utils -->
+<script type="text/javascript" src="global.js"></script>
+
+<!-- window onload -->
+<script type="text/javascript">
+
+function DrawAggrCharts()
+{
+ // get refresh url
+ lookback = aggrLookBack;
+ var url = DataUrl();
+
+ // fetch data, draw charts
+ try
+ {
+ var connect = new XMLHttpRequest();
+ connect.onreadystatechange = function() {
+ if(connect.readyState==4 && connect.status==200) {
+
+console.log(url);
+ aggrData = connect.response;
+ var pts = JSON.parse(aggrData);
+ aggrDataPoints = new Array();
+ for(var i=0; i < pts.length; i++) aggrDataPoints.push(pts[i]);
+ DrawCPUChart();
+ DrawRAMChart();
+ DrawHDDChart();
+ //DrawImpressionsChart();
+ delete aggrData;
+ }
+ }
+ connect.open('GET', url, true);
+ connect.send(null);
+ } catch(e) {
+ }
+ aggrLookBack += 30;
+}
+
+function DrawContCharts()
+{
+ // get refresh url
+ lookback = contLookBack;
+ var url = DataUrl();
+ //document.getElementById('chart_div').innerHTML = url;
+
+ // fetch data, draw charts
+ try
+ {
+ var connect = new XMLHttpRequest();
+ connect.onreadystatechange = function() {
+ if(connect.readyState==4 && connect.status==200) {
+ contData = connect.response;
+ var newPts = JSON.parse(contData);
+ contDataPoints = new Array();
+ for(var i=0; i < newPts.length; i++) contDataPoints.push(newPts[i]);
+ DrawCtrChart() ;
+ DrawMarginChart();
+ delete contData;
+ delete newPts;
+ }
+ }
+ connect.open('GET', url, true);
+ connect.send(null);
+ } catch(e) {
+ }
+ contLookBack += contRefresh;
+}
+
+window.onload = function() {
+
+ // Initialize global
+ InitializeGlobal();
+
+ // Inituialize form fields
+ if (params['customer']) document.getElementById('customer').value = params['customer'];
+ if (params['product']) document.getElementById('product').value = params['product'];
+ if (params['os']) document.getElementById('os').value = params['os'];
+ if (params['software1']) document.getElementById('software1').value = params['software1'];
+ if (params['software2']) document.getElementById('software2').value = params['software2'];
+ if (params['software3']) document.getElementById('software3').value = params['software3'];
+ if (params['refresh'])
+ {
+ document.getElementById('refresh').value = params['refresh'];
+ } else {
+ document.getElementById('refresh').value = 5;
+ }
+ if (params['lookback'])
+ {
+ document.getElementById('lookback').value = params['lookback'];
+ } else {
+ document.getElementById('lookback').value = 6;
+ }
+
+ // draw charts
+ DrawAggrCharts();
+ //DrawContCharts();
+ setInterval(DrawAggrCharts, 30000);
+ //setInterval(DrawContCharts, contRefresh * 1000);
+};
+
+</script>
+
+</head>
+<body>
+
+ <div id="header">
+ <ul class="dashboard-modes">
+ <li>
+ <a href="#" class="active">Machine Generated Data Example </a>
+ </li>
+ </ul>
+
+ </div>
+
+ <div id="main">
+ <div id="pagecontent">
+ <div class="dashboardMgr">
+ <div class="inner" style="">
+ <h2 class="title">View Real Time Data Charts</h2>
+ <form method="GET" action="index.php">
+
+ <label for="customer">Customer ID:</label>
+ <select name="customer" id="customer" style="width:200px;">
+ <option value="">ALL</option>
+ <?php
+ for ($i = 1; $i <= 5; $i++) {
+ print "<option value=\"$i\">Customer $i</option>\n";
+ }
+ ?>
+ </select>
+
+ <label for="">Product ID:</label>
+ <select name="product" id="product" style="width:200px;">
+ <option value="">ALL</option>
+ <?php
+ for ($i = 4; $i <= 6; $i++) {
+ print "<option value=\"$i\">Product $i</option>\n";
+ }
+ ?>
+ </select>
+
+ <label for="">Product OS:</label>
+ <select name="os" id="os" style="width:200px;">
+ <option value="">ALL</option>
+ <?php
+ for ($i = 10; $i <= 12; $i++) {
+ print "<option value=\"$i\">OS $i</option>\n";
+ }
+ ?>
+ </select>
+
+ <label for="software1">Software1 Ver:</label>
+ <select name="software1" id="software1" style="width:200px;">
+ <option value="">ALL</option>
+ <?php
+ for ($i = 10; $i <= 12; $i++) {
+ print "<option value=\"$i\">Software1 Version $i</option>\n";
+ }
+ ?>
+ </select>
+
+ <label for="software2">Software2 Ver:</label>
+ <select name="software2" id="software2" style="width:200px;">
+ <option value="">ALL</option>
+ <?php
+ for ($i = 12; $i <= 14; $i++) {
+ print "<option value=\"$i\">Software2 Version $i</option>\n";
+ }
+ ?>
+ </select>
+
+ <label for="software3">Software3 Ver:</label>
+ <select name="software3" id="software3" style="width:200px;">
+ <option value="">ALL</option>
+ <?php
+ for ($i = 4; $i <= 6; $i++) {
+ print "<option value=\"$i\">Software3 Version $i</option>\n";
+ }
+ ?>
+ </select>
+
+ <label for="">Refresh Interval:</label>
+ <div class="input-append">
+ <input type="text" name="refresh" id="refresh" class="input-small"/>
+ <span class="add-on">Secs</span>
+ </div>
+
+
+ <label for="">Look Back:</label>
+ <div class="input-append">
+ <input type="text" name="lookback" id="lookback" class="input-small"/>
+ <span class="add-on">Hours</span>
+ </div>
+
+ <input type="submit" value="submit" class="btn btn-primary" />
+
+ </form>
+ </div>
+ <div class="collapser-container">
+ <div class="collapser">
+ <div class="collapse-dot"></div>
+ <div class="collapse-dot"></div>
+ <div class="collapse-dot"></div>
+ </div>
+ </div>
+ </div>
+ <div class="dashboardMain">
+
+ <!-- <table><tbody>
+ <tr>
+ <td><div id="chart_div"></div></td>
+ <td><div id="chart1_div" ></div></td>
+ </tr>
+ <tr>
+ <td><div id="chart2_div" ></div></td>
+ <td><div id="chart3_div" ></div></td>
+ </tr>
+ <tr>
+ <td><div id="chart4_div" ></div></td>
+ <td><div id="chart5_div" ></div></td>
+ </tr>
+ </tr></tbody></table> -->
+ <div class="chart-ctnr" id="chart_div"></div>
+ <div class="chart-ctnr" id="chart1_div" ></div>
+ <div class="chart-ctnr" id="chart2_div" ></div>
+<!-- <div class="chart-ctnr" id="chart3_div" ></div>
+ <div class="chart-ctnr" id="chart4_div" ></div>
+ <div class="chart-ctnr" id="chart5_div" ></div> -->
+ </div>
+</body>
+</html>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/machinedata/src/main/html/json.php
----------------------------------------------------------------------
diff --git a/examples/machinedata/src/main/html/json.php b/examples/machinedata/src/main/html/json.php
new file mode 100644
index 0000000..75a7117
--- /dev/null
+++ b/examples/machinedata/src/main/html/json.php
@@ -0,0 +1,96 @@
+<?php
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+header("Content-type: application/json");
+$redis = new Redis();
+$redis->connect('localhost');
+$redis->select(15);
+$from = $_GET['from'];
+$bucket = $_GET['bucket'];
+$customer = $_GET['customer'];
+$product = $_GET['product'];
+$os = $_GET['os'];
+$software1 = $_GET['software1'];
+$software2 = $_GET['software2'];
+$software3 = $_GET['software3'];
+
+switch ($bucket) {
+case 'D':
+ $format = 'Ymd';
+ $incr = 60 * 60 * 24;
+ break;
+case 'h':
+ $format = 'YmdH';
+ $incr = 60 * 60;
+ break;
+case 'm':
+ $format = 'YmdHi';
+ $incr = 60;
+ break;
+default:
+ break;
+}
+
+$arr = array();
+if ($customer != '') {
+ $arr[] = "0:".$customer;
+}
+if ($product != '') {
+ $arr[] = "1:".$product;
+}
+if ($os != '') {
+ $arr[] = "2:".$os;
+}
+if ($software1 != '') {
+ $arr[] = "3:".$software1;
+}
+if ($software2 != '') {
+ $arr[] = "4:".$software2;
+}
+if ($software3 != '') {
+ $arr[] = "5:".$software3;
+}
+$subpattern = "";
+if (count($arr) != 0) {
+ $subpattern = join("|", $arr);
+}
+
+$result = array();
+
+while ($from < time()) {
+ $date = gmdate($format, $from);
+ if ($subpattern != '') {
+ $key = $bucket . '|' . $date . '|' . $subpattern;
+ } else {
+ $key = $bucket . '|' . $date ;
+ }
+ $hash = $redis->hGetAll($key);
+ if ($hash) {
+ $cpu = $hash['cpu'];
+ $ram = $hash['ram'];
+ $hdd = $hash['hdd'];
+ $result[] = array('timestamp'=> $from * 1000, 'cpu'=>$cpu, 'ram'=>$ram, 'hdd'=>$hdd);
+ }
+ $from += $incr;
+}
+
+array_pop($result);
+print json_encode($result);
+
+?>
[14/30] apex-malhar git commit: Renamed demos to examples. Packages
and artifactid names are changed as suggested.
Posted by th...@apache.org.
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/echoserver/src/main/java/org/apache/apex/examples/echoserver/MessageReceiver.java
----------------------------------------------------------------------
diff --git a/examples/echoserver/src/main/java/org/apache/apex/examples/echoserver/MessageReceiver.java b/examples/echoserver/src/main/java/org/apache/apex/examples/echoserver/MessageReceiver.java
new file mode 100644
index 0000000..34a2c56
--- /dev/null
+++ b/examples/echoserver/src/main/java/org/apache/apex/examples/echoserver/MessageReceiver.java
@@ -0,0 +1,156 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.echoserver;
+
+import java.io.IOException;
+import java.net.SocketAddress;
+import java.nio.ByteBuffer;
+import java.nio.channels.DatagramChannel;
+import java.nio.channels.SelectionKey;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.datatorrent.api.Context;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.InputOperator;
+
+/**
+ * @since 2.1.0
+ */
+public class MessageReceiver implements InputOperator, NetworkManager.ChannelListener<DatagramChannel>
+{
+ private static final Logger logger = LoggerFactory.getLogger(MessageReceiver.class);
+
+ private transient NetworkManager.ChannelAction<DatagramChannel> action;
+
+ //Need the sender info, using a packet for now instead of the buffer
+ private transient ByteBuffer buffer;
+ //private transient DatagramPacket packet;
+
+ private int port = 9000;
+ private int maxMesgSize = 512;
+ private int inactiveWait = 10;
+ private boolean readReady = false;
+
+ @Override
+ public void emitTuples()
+ {
+ boolean emitData = false;
+ if (readReady) {
+ //DatagramSocket socket = action.channelConfiguration.socket;
+ try {
+ //socket.receive(packet);
+ DatagramChannel channel = action.channelConfiguration.channel;
+ SocketAddress address = channel.receive(buffer);
+ if (address != null) {
+ /*
+ StringBuilder sb = new StringBuilder();
+ buffer.rewind();
+ while (buffer.hasRemaining()) {
+ sb.append(buffer.getChar());
+ }
+ String mesg = sb.toString();
+ */
+ buffer.flip();
+ String mesg = new String(buffer.array(), 0, buffer.limit());
+ logger.info("Message {}", mesg);
+ Message message = new Message();
+ message.message = mesg;
+ message.socketAddress = address;
+ messageOutput.emit(message);
+ emitData = true;
+ buffer.clear();
+ }
+ //String mesg = new String(packet.getData(), packet.getOffset(), packet.getLength());
+ } catch (IOException e) {
+ throw new RuntimeException("Error reading from channel", e);
+ }
+ // Even if we miss a readReady because of not locking we will get it again immediately
+ readReady = false;
+ }
+ if (!emitData) {
+ synchronized (buffer) {
+ try {
+ if (!readReady) {
+ buffer.wait(inactiveWait);
+ }
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+ }
+
+ @Override
+ public void beginWindow(long l)
+ {
+
+ }
+
+ @Override
+ public void endWindow()
+ {
+
+ }
+
+ public final transient DefaultOutputPort<Message> messageOutput = new DefaultOutputPort<Message>();
+
+ @Override
+ public void setup(Context.OperatorContext context)
+ {
+ try {
+ //byte[] mesgData = new byte[maxMesgSize];
+ //packet = new DatagramPacket(mesgData, maxMesgSize);
+ buffer = ByteBuffer.allocate(maxMesgSize);
+ action = NetworkManager.getInstance().registerAction(port, NetworkManager.ConnectionType.UDP, this, SelectionKey.OP_READ);
+ } catch (IOException e) {
+ throw new RuntimeException("Error initializing receiver", e);
+ }
+ }
+
+ @Override
+ public void teardown()
+ {
+ try {
+ NetworkManager.getInstance().unregisterAction(action);
+ } catch (Exception e) {
+ throw new RuntimeException("Error shutting down receiver", e);
+ }
+ }
+
+ @Override
+ public void ready(NetworkManager.ChannelAction<DatagramChannel> action, int readyOps)
+ {
+ synchronized (buffer) {
+ readReady = true;
+ buffer.notify();
+ }
+ }
+
+ public int getPort()
+ {
+ return port;
+ }
+
+ public void setPort(int port)
+ {
+ this.port = port;
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/echoserver/src/main/java/org/apache/apex/examples/echoserver/MessageResponder.java
----------------------------------------------------------------------
diff --git a/examples/echoserver/src/main/java/org/apache/apex/examples/echoserver/MessageResponder.java b/examples/echoserver/src/main/java/org/apache/apex/examples/echoserver/MessageResponder.java
new file mode 100644
index 0000000..d34bf3e
--- /dev/null
+++ b/examples/echoserver/src/main/java/org/apache/apex/examples/echoserver/MessageResponder.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.echoserver;
+
+import java.io.IOException;
+import java.net.SocketAddress;
+import java.nio.ByteBuffer;
+import java.nio.channels.DatagramChannel;
+
+import com.datatorrent.api.Context;
+import com.datatorrent.api.DefaultInputPort;
+import com.datatorrent.common.util.BaseOperator;
+
+/**
+ * @since 2.1.0
+ */
+public class MessageResponder extends BaseOperator
+{
+
+ private String responseHeader = "Response: ";
+
+ private int port = 9000;
+ private int maxMesgSize = 512;
+ private transient NetworkManager.ChannelAction<DatagramChannel> action;
+ private transient ByteBuffer buffer;
+
+ public final transient DefaultInputPort<Message> messageInput = new DefaultInputPort<Message>()
+ {
+ @Override
+ public void process(Message message)
+ {
+ String sendMesg = responseHeader + message.message;
+ SocketAddress address = message.socketAddress;
+ buffer.put(sendMesg.getBytes());
+ buffer.flip();
+ try {
+ action.channelConfiguration.channel.send(buffer, address);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ buffer.clear();
+ }
+ };
+
+ @Override
+ public void setup(Context.OperatorContext context)
+ {
+ try {
+ buffer = ByteBuffer.allocate(maxMesgSize);
+ action = NetworkManager.getInstance().registerAction(port, NetworkManager.ConnectionType.UDP, null, 0);
+ } catch (IOException e) {
+ throw new RuntimeException("Error initializer responder", e);
+ }
+ }
+
+ @Override
+ public void teardown()
+ {
+ try {
+ NetworkManager.getInstance().unregisterAction(action);
+ } catch (Exception e) {
+ throw new RuntimeException("Error shutting down responder", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/echoserver/src/main/java/org/apache/apex/examples/echoserver/NetworkManager.java
----------------------------------------------------------------------
diff --git a/examples/echoserver/src/main/java/org/apache/apex/examples/echoserver/NetworkManager.java b/examples/echoserver/src/main/java/org/apache/apex/examples/echoserver/NetworkManager.java
new file mode 100644
index 0000000..88cf621
--- /dev/null
+++ b/examples/echoserver/src/main/java/org/apache/apex/examples/echoserver/NetworkManager.java
@@ -0,0 +1,249 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.echoserver;
+
+import java.io.IOException;
+import java.net.DatagramSocket;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.net.SocketAddress;
+import java.nio.channels.DatagramChannel;
+import java.nio.channels.SelectableChannel;
+import java.nio.channels.SelectionKey;
+import java.nio.channels.Selector;
+import java.nio.channels.SocketChannel;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentLinkedQueue;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @since 2.1.0
+ */
+public class NetworkManager implements Runnable
+{
+ private static final Logger logger = LoggerFactory.getLogger(NetworkManager.class);
+
+ public static enum ConnectionType
+ {
+ TCP,
+ UDP
+ }
+
+ private static NetworkManager _instance;
+ private Selector selector;
+
+ private volatile boolean doRun = false;
+ private Thread selThread;
+ private long selTimeout = 1000;
+ private volatile Exception selEx;
+
+ private Map<ConnectionInfo, ChannelConfiguration> channels;
+ private Map<SelectableChannel, ChannelConfiguration> channelConfigurations;
+
+ public static NetworkManager getInstance() throws IOException
+ {
+ if (_instance == null) {
+ synchronized (NetworkManager.class) {
+ if (_instance == null) {
+ _instance = new NetworkManager();
+ }
+ }
+ }
+ return _instance;
+ }
+
+ private NetworkManager() throws IOException
+ {
+ channels = new HashMap<ConnectionInfo, ChannelConfiguration>();
+ channelConfigurations = new HashMap<SelectableChannel, ChannelConfiguration>();
+ }
+
+ public synchronized <T extends SelectableChannel> ChannelAction<T> registerAction(int port, ConnectionType type, ChannelListener<T> listener, int ops) throws IOException
+ {
+ boolean startProc = (channels.size() == 0);
+ SelectableChannel channel = null;
+ SocketAddress address = new InetSocketAddress(port);
+ ConnectionInfo connectionInfo = new ConnectionInfo();
+ connectionInfo.address = address;
+ connectionInfo.connectionType = type;
+ ChannelConfiguration channelConfiguration = channels.get(connectionInfo);
+ if (channelConfiguration == null) {
+ Object socket = null;
+ if (type == ConnectionType.TCP) {
+ SocketChannel schannel = SocketChannel.open();
+ schannel.configureBlocking(false);
+ Socket ssocket = schannel.socket();
+ ssocket.bind(address);
+ socket = ssocket;
+ channel = schannel;
+ } else if (type == ConnectionType.UDP) {
+ DatagramChannel dchannel = DatagramChannel.open();
+ dchannel.configureBlocking(false);
+ DatagramSocket dsocket = dchannel.socket();
+ dsocket.bind(address);
+ socket = dsocket;
+ channel = dchannel;
+ }
+ if (channel == null) {
+ throw new IOException("Unsupported connection type");
+ }
+ channelConfiguration = new ChannelConfiguration();
+ channelConfiguration.actions = new ConcurrentLinkedQueue<ChannelAction>();
+ channelConfiguration.channel = channel;
+ channelConfiguration.connectionInfo = connectionInfo;
+ channels.put(connectionInfo, channelConfiguration);
+ channelConfigurations.put(channel, channelConfiguration);
+ } else {
+ channel = channelConfiguration.channel;
+ }
+ ChannelAction channelAction = new ChannelAction();
+ channelAction.channelConfiguration = channelConfiguration;
+ channelAction.listener = listener;
+ channelAction.ops = ops;
+ channelConfiguration.actions.add(channelAction);
+ if (startProc) {
+ startProcess();
+ }
+ if (listener != null) {
+ channel.register(selector, ops);
+ }
+ return channelAction;
+ }
+
+ public synchronized void unregisterAction(ChannelAction action) throws IOException, InterruptedException
+ {
+ ChannelConfiguration channelConfiguration = action.channelConfiguration;
+ SelectableChannel channel = channelConfiguration.channel;
+ if (channelConfiguration != null) {
+ channelConfiguration.actions.remove(action);
+ if (channelConfiguration.actions.size() == 0) {
+ ConnectionInfo connectionInfo = channelConfiguration.connectionInfo;
+ channelConfigurations.remove(channel);
+ channels.remove(connectionInfo);
+ channel.close();
+ }
+ }
+ if (channels.size() == 0) {
+ stopProcess();
+ }
+ }
+
+ private void startProcess() throws IOException
+ {
+ selector = Selector.open();
+ doRun = true;
+ selThread = new Thread(this);
+ selThread.start();
+ }
+
+ private void stopProcess() throws InterruptedException, IOException
+ {
+ doRun = false;
+ selThread.join();
+ selector.close();
+ }
+
+ @Override
+ public void run()
+ {
+ try {
+ while (doRun) {
+ int keys = selector.select(selTimeout);
+ if (keys > 0) {
+ Set<SelectionKey> selectionKeys = selector.selectedKeys();
+ for (SelectionKey selectionKey : selectionKeys) {
+ int readyOps = selectionKey.readyOps();
+ ChannelConfiguration channelConfiguration = channelConfigurations.get(selectionKey.channel());
+ Collection<ChannelAction> actions = channelConfiguration.actions;
+ for (ChannelAction action : actions) {
+ if (((readyOps & action.ops) != 0) && (action.listener != null)) {
+ action.listener.ready(action, readyOps);
+ }
+ }
+ }
+ selectionKeys.clear();
+ }
+ }
+ } catch (IOException e) {
+ logger.error("Error in select", e);
+ selEx = e;
+ }
+ }
+
+ public static interface ChannelListener<T extends SelectableChannel>
+ {
+ public void ready(ChannelAction<T> action, int readyOps);
+ }
+
+ public static class ChannelConfiguration<T extends SelectableChannel>
+ {
+ public T channel;
+ public ConnectionInfo connectionInfo;
+ public Collection<ChannelAction> actions;
+ }
+
+ public static class ChannelAction<T extends SelectableChannel>
+ {
+ public ChannelConfiguration<T> channelConfiguration;
+ public ChannelListener<T> listener;
+ public int ops;
+ }
+
+ private static class ConnectionInfo
+ {
+ public SocketAddress address;
+ public ConnectionType connectionType;
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ ConnectionInfo that = (ConnectionInfo)o;
+
+ if (connectionType != that.connectionType) {
+ return false;
+ }
+ if (!address.equals(that.address)) {
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override
+ public int hashCode()
+ {
+ int result = address.hashCode();
+ result = 31 * result + connectionType.hashCode();
+ return result;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/echoserver/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/examples/echoserver/src/main/resources/META-INF/properties.xml b/examples/echoserver/src/main/resources/META-INF/properties.xml
new file mode 100644
index 0000000..0822f4f
--- /dev/null
+++ b/examples/echoserver/src/main/resources/META-INF/properties.xml
@@ -0,0 +1,38 @@
+<?xml version="1.0"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<configuration>
+ <!--
+ <property>
+ <name>dt.application.{appName}.operator.{opName}.prop.{propName}</name>
+ <value>some-default-value (if value is not specified, it is required from the user or custom config when launching)</value>
+ </property>
+ -->
+ <property>
+ <name>dt.attr.MASTER_MEMORY_MB</name>
+ <value>1024</value>
+ </property>
+ <property>
+ <name>dt.application.MyFirstApplication.operator.console.prop.stringFormat</name>
+ <value>hello world: %s</value>
+ </property>
+</configuration>
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/echoserver/src/test/java/org/apache/apex/examples/echoserver/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/echoserver/src/test/java/org/apache/apex/examples/echoserver/ApplicationTest.java b/examples/echoserver/src/test/java/org/apache/apex/examples/echoserver/ApplicationTest.java
new file mode 100644
index 0000000..c4a3ad4
--- /dev/null
+++ b/examples/echoserver/src/test/java/org/apache/apex/examples/echoserver/ApplicationTest.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.echoserver;
+
+import java.io.IOException;
+
+import javax.validation.ConstraintViolationException;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.LocalMode;
+
+/**
+ * Test the DAG declaration in local mode.
+ */
+public class ApplicationTest
+{
+
+ @Test
+ public void testApplication() throws IOException, Exception
+ {
+ try {
+ LocalMode lma = LocalMode.newInstance();
+ Configuration conf = new Configuration(false);
+ conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
+ lma.prepareDAG(new Application(), conf);
+ LocalMode.Controller lc = lma.getController();
+ lc.run(10000); // runs for 10 seconds and quits
+ } catch (ConstraintViolationException e) {
+ Assert.fail("constraint violations: " + e.getConstraintViolations());
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/echoserver/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/examples/echoserver/src/test/resources/log4j.properties b/examples/echoserver/src/test/resources/log4j.properties
new file mode 100644
index 0000000..cf0d19e
--- /dev/null
+++ b/examples/echoserver/src/test/resources/log4j.properties
@@ -0,0 +1,43 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+log4j.rootLogger=DEBUG,CONSOLE
+
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+log4j.appender.CONSOLE.threshold=${test.log.console.threshold}
+test.log.console.threshold=DEBUG
+
+log4j.appender.RFA=org.apache.log4j.RollingFileAppender
+log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
+log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+log4j.appender.RFA.File=/tmp/app.log
+
+# to enable, add SYSLOG to rootLogger
+log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender
+log4j.appender.SYSLOG.syslogHost=127.0.0.1
+log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout
+log4j.appender.SYSLOG.layout.conversionPattern=${dt.cid} %-5p [%t] %c{2} %x - %m%n
+log4j.appender.SYSLOG.Facility=LOCAL1
+
+log4j.logger.org=info
+#log4j.logger.org.apache.commons.beanutils=warn
+log4j.logger.com.datatorrent=debug
+log4j.logger.org.apache.apex=debug
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/frauddetect/pom.xml
----------------------------------------------------------------------
diff --git a/examples/frauddetect/pom.xml b/examples/frauddetect/pom.xml
new file mode 100644
index 0000000..36d4153
--- /dev/null
+++ b/examples/frauddetect/pom.xml
@@ -0,0 +1,49 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>malhar-examples-frauddetect</artifactId>
+ <packaging>jar</packaging>
+
+ <name>Apache Apex Malhar Fraud Detect Example</name>
+ <description>Apex example application that demonstrates real-time pattern detection in the incoming data and alerting. The example processes streaming credit card transactions and looks for fraudulent transactions.</description>
+
+ <parent>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-examples</artifactId>
+ <version>3.7.0-SNAPSHOT</version>
+ </parent>
+
+ <properties>
+ <skipTests>true</skipTests>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.mongodb</groupId>
+ <artifactId>mongo-java-driver</artifactId>
+ <version>2.10.1</version>
+ </dependency>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/frauddetect/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/examples/frauddetect/src/assemble/appPackage.xml b/examples/frauddetect/src/assemble/appPackage.xml
new file mode 100644
index 0000000..4138cf2
--- /dev/null
+++ b/examples/frauddetect/src/assemble/appPackage.xml
@@ -0,0 +1,59 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+ <id>appPackage</id>
+ <formats>
+ <format>jar</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>${basedir}/target/</directory>
+ <outputDirectory>/app</outputDirectory>
+ <includes>
+ <include>${project.artifactId}-${project.version}.jar</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/target/deps</directory>
+ <outputDirectory>/lib</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/site/conf</directory>
+ <outputDirectory>/conf</outputDirectory>
+ <includes>
+ <include>*.xml</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/META-INF</directory>
+ <outputDirectory>/META-INF</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/app</directory>
+ <outputDirectory>/app</outputDirectory>
+ </fileSet>
+ </fileSets>
+
+</assembly>
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/Application.java
----------------------------------------------------------------------
diff --git a/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/Application.java b/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/Application.java
new file mode 100644
index 0000000..73c38ef
--- /dev/null
+++ b/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/Application.java
@@ -0,0 +1,151 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.frauddetect;
+
+import java.io.Serializable;
+import java.net.URI;
+import org.apache.apex.examples.frauddetect.operator.HdfsStringOutputOperator;
+import org.apache.apex.examples.frauddetect.operator.MongoDBOutputOperator;
+import org.apache.hadoop.conf.Configuration;
+import com.datatorrent.api.Context;
+import com.datatorrent.api.Context.DAGContext;
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.lib.io.ConsoleOutputOperator;
+import com.datatorrent.lib.io.PubSubWebSocketInputOperator;
+import com.datatorrent.lib.io.PubSubWebSocketOutputOperator;
+import com.datatorrent.lib.math.RangeKeyVal;
+import com.datatorrent.lib.multiwindow.SimpleMovingAverage;
+import com.datatorrent.lib.util.BaseKeyValueOperator;
+import com.datatorrent.lib.util.KeyValPair;
+import com.datatorrent.netlet.util.DTThrowable;
+
+
+/**
+ * Fraud detection application
+ *
+ * @since 0.9.0
+ */
+@ApplicationAnnotation(name = "FraudDetectExample")
+public class Application implements StreamingApplication
+{
+
+ public PubSubWebSocketInputOperator getPubSubWebSocketInputOperator(String name, DAG dag, URI duri, String topic) throws Exception
+ {
+ PubSubWebSocketInputOperator reqin = dag.addOperator(name, new PubSubWebSocketInputOperator());
+ reqin.setUri(duri);
+ reqin.setTopic(topic);
+ return reqin;
+ }
+
+ public PubSubWebSocketOutputOperator getPubSubWebSocketOutputOperator(String name, DAG dag, URI duri, String topic) throws Exception
+ {
+ PubSubWebSocketOutputOperator out = dag.addOperator(name, new PubSubWebSocketOutputOperator());
+ out.setUri(duri);
+ return out;
+ }
+
+ public HdfsStringOutputOperator getHdfsOutputOperator(String name, DAG dag, String folderName)
+ {
+ HdfsStringOutputOperator oper = dag.addOperator("hdfs", HdfsStringOutputOperator.class);
+ oper.setFilePath(folderName);
+ oper.setMaxLength(1024 * 1024 * 1024);
+ return oper;
+ }
+
+ public ConsoleOutputOperator getConsoleOperator(String name, DAG dag, String prefix, String format)
+ {
+ ConsoleOutputOperator oper = dag.addOperator(name, ConsoleOutputOperator.class);
+ // oper.setStringFormat(prefix + ": " + format);
+ return oper;
+ }
+
+ public static class KeyPartitionCodec<K, V> extends BaseKeyValueOperator.DefaultPartitionCodec<K,V> implements Serializable
+ {
+ private static final long serialVersionUID = 201410031623L;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+
+ try {
+ String gatewayAddress = dag.getValue(DAGContext.GATEWAY_CONNECT_ADDRESS);
+ if (gatewayAddress == null) {
+ gatewayAddress = "localhost:9090";
+ }
+ URI duri = URI.create("ws://" + gatewayAddress + "/pubsub");
+
+ PubSubWebSocketInputOperator userTxWsInput = getPubSubWebSocketInputOperator("userTxInput", dag, duri, "examples.app.frauddetect.submitTransaction");
+ PubSubWebSocketOutputOperator ccUserAlertWsOutput = getPubSubWebSocketOutputOperator("ccUserAlertQueryOutput", dag, duri, "examples.app.frauddetect.fraudAlert");
+ PubSubWebSocketOutputOperator avgUserAlertwsOutput = getPubSubWebSocketOutputOperator("avgUserAlertQueryOutput", dag, duri, "examples.app.frauddetect.fraudAlert");
+ PubSubWebSocketOutputOperator binUserAlertwsOutput = getPubSubWebSocketOutputOperator("binUserAlertOutput", dag, duri, "examples.app.frauddetect.fraudAlert");
+ PubSubWebSocketOutputOperator txSummaryWsOutput = getPubSubWebSocketOutputOperator("txSummaryWsOutput", dag, duri, "examples.app.frauddetect.txSummary");
+ SlidingWindowSumKeyVal<KeyValPair<MerchantKey, String>, Integer> smsOperator = dag.addOperator("movingSum", SlidingWindowSumKeyVal.class);
+
+ MerchantTransactionGenerator txReceiver = dag.addOperator("txReceiver", MerchantTransactionGenerator.class);
+ MerchantTransactionInputHandler txInputHandler = dag.addOperator("txInputHandler", new MerchantTransactionInputHandler());
+ BankIdNumberSamplerOperator binSampler = dag.addOperator("bankInfoFraudDetector", BankIdNumberSamplerOperator.class);
+
+ MerchantTransactionBucketOperator txBucketOperator = dag.addOperator("txFilter", MerchantTransactionBucketOperator.class);
+ RangeKeyVal rangeOperator = dag.addOperator("rangePerMerchant", new RangeKeyVal<MerchantKey, Long>());
+ SimpleMovingAverage<MerchantKey, Long> smaOperator = dag.addOperator("smaPerMerchant", SimpleMovingAverage.class);
+ TransactionStatsAggregator txStatsAggregator = dag.addOperator("txStatsAggregator", TransactionStatsAggregator.class);
+ AverageAlertingOperator avgAlertingOperator = dag.addOperator("avgAlerter", AverageAlertingOperator.class);
+ CreditCardAmountSamplerOperator ccSamplerOperator = dag.addOperator("amountFraudDetector", CreditCardAmountSamplerOperator.class);
+ HdfsStringOutputOperator hdfsOutputOperator = getHdfsOutputOperator("hdfsOutput", dag, "fraud");
+
+ MongoDBOutputOperator mongoTxStatsOperator = dag.addOperator("mongoTxStatsOutput", MongoDBOutputOperator.class);
+ MongoDBOutputOperator mongoBinAlertsOperator = dag.addOperator("mongoBinAlertsOutput", MongoDBOutputOperator.class);
+ MongoDBOutputOperator mongoCcAlertsOperator = dag.addOperator("mongoCcAlertsOutput", MongoDBOutputOperator.class);
+ MongoDBOutputOperator mongoAvgAlertsOperator = dag.addOperator("mongoAvgAlertsOutput", MongoDBOutputOperator.class);
+
+ dag.addStream("userTxStream", userTxWsInput.outputPort, txInputHandler.userTxInputPort);
+ dag.addStream("transactions", txReceiver.txOutputPort, txBucketOperator.inputPort).setLocality(DAG.Locality.CONTAINER_LOCAL);
+ dag.addStream("txData", txReceiver.txDataOutputPort, hdfsOutputOperator.input); // dump all tx into Hdfs
+ dag.addStream("userTransactions", txInputHandler.txOutputPort, txBucketOperator.txUserInputPort);
+ dag.addStream("bankInfoData", txBucketOperator.binCountOutputPort, smsOperator.data);
+ dag.addStream("bankInfoCount", smsOperator.integerSum, binSampler.txCountInputPort);
+ dag.addStream("filteredTransactions", txBucketOperator.txOutputPort, rangeOperator.data, smaOperator.data, avgAlertingOperator.txInputPort);
+
+ KeyPartitionCodec<MerchantKey, Long> txCodec = new KeyPartitionCodec<MerchantKey, Long>();
+ dag.setInputPortAttribute(rangeOperator.data, Context.PortContext.STREAM_CODEC, txCodec);
+ dag.setInputPortAttribute(smaOperator.data, Context.PortContext.STREAM_CODEC, txCodec);
+ dag.setInputPortAttribute(avgAlertingOperator.txInputPort, Context.PortContext.STREAM_CODEC, txCodec);
+
+ dag.addStream("creditCardData", txBucketOperator.ccAlertOutputPort, ccSamplerOperator.inputPort);
+ dag.addStream("txnSummaryData", txBucketOperator.summaryTxnOutputPort, txSummaryWsOutput.input);
+ dag.addStream("smaAlerts", smaOperator.doubleSMA, avgAlertingOperator.smaInputPort);
+ dag.addStream("binAlerts", binSampler.countAlertOutputPort, mongoBinAlertsOperator.inputPort);
+ dag.addStream("binAlertsNotification", binSampler.countAlertNotificationPort, binUserAlertwsOutput.input);
+ dag.addStream("rangeData", rangeOperator.range, txStatsAggregator.rangeInputPort);
+ dag.addStream("smaData", smaOperator.longSMA, txStatsAggregator.smaInputPort);
+ dag.addStream("txStatsOutput", txStatsAggregator.txDataOutputPort, mongoTxStatsOperator.inputPort);
+ dag.addStream("avgAlerts", avgAlertingOperator.avgAlertOutputPort, mongoAvgAlertsOperator.inputPort);
+ dag.addStream("avgAlertsNotification", avgAlertingOperator.avgAlertNotificationPort, avgUserAlertwsOutput.input);
+ dag.addStream("ccAlerts", ccSamplerOperator.ccAlertOutputPort, mongoCcAlertsOperator.inputPort);
+ dag.addStream("ccAlertsNotification", ccSamplerOperator.ccAlertNotificationPort, ccUserAlertWsOutput.input);
+
+ } catch (Exception exc) {
+ DTThrowable.rethrow(exc);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/AverageAlertData.java
----------------------------------------------------------------------
diff --git a/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/AverageAlertData.java b/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/AverageAlertData.java
new file mode 100644
index 0000000..6aca64d
--- /dev/null
+++ b/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/AverageAlertData.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.frauddetect;
+
+/**
+ * POJO to capture average alert data.
+ *
+ * @since 0.9.0
+ */
+public class AverageAlertData
+{
+ public String merchantId;
+ public int terminalId;
+ public int zipCode;
+ public MerchantTransaction.MerchantType merchantType;
+ public long amount;
+ public double lastSmaValue;
+ public double change;
+ public boolean userGenerated;
+ public long time;
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/AverageAlertingOperator.java
----------------------------------------------------------------------
diff --git a/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/AverageAlertingOperator.java b/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/AverageAlertingOperator.java
new file mode 100644
index 0000000..1b1b64a
--- /dev/null
+++ b/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/AverageAlertingOperator.java
@@ -0,0 +1,186 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.frauddetect;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import javax.validation.constraints.NotNull;
+
+import org.codehaus.jackson.JsonFactory;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.apex.examples.frauddetect.util.JsonUtils;
+import org.apache.commons.lang.mutable.MutableDouble;
+
+import com.datatorrent.api.DefaultInputPort;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.common.util.BaseOperator;
+
+import com.datatorrent.lib.util.KeyValPair;
+
+/**
+ * Generate an alert if the current transaction amount received on tx input port for the given key is greater by n %
+ * than the SMA of the last application window as received on the SMA input port.
+ *
+ * @since 0.9.0
+ */
+public class AverageAlertingOperator extends BaseOperator
+{
+ private static final Logger Log = LoggerFactory.getLogger(AverageAlertingOperator.class);
+ private final transient JsonFactory jsonFactory = new JsonFactory();
+ private final transient ObjectMapper mapper = new ObjectMapper(jsonFactory);
+ private Map<MerchantKey, MutableDouble> lastSMAMap = new HashMap<MerchantKey, MutableDouble>();
+ private Map<MerchantKey, MutableDouble> currentSMAMap = new HashMap<MerchantKey, MutableDouble>();
+ private List<AverageAlertData> alerts = new ArrayList<AverageAlertData>();
+ @NotNull
+ private int threshold;
+ private static final String brickMortarAlertMsg = "Transaction amount %d exceeded by %f (last SMA %f) for Merchant %s at Terminal %d!";
+ private static final String internetAlertMsg = "Transaction amount %d exceeded by %f (last SMA %f) for Merchant %s!";
+ public final transient DefaultOutputPort<String> avgAlertOutputPort = new DefaultOutputPort<String>();
+ public final transient DefaultOutputPort<Map<String, Object>> avgAlertNotificationPort = new DefaultOutputPort<Map<String, Object>>();
+ public final transient DefaultInputPort<KeyValPair<MerchantKey, Double>> smaInputPort =
+ new DefaultInputPort<KeyValPair<MerchantKey, Double>>()
+ {
+ @Override
+ public void process(KeyValPair<MerchantKey, Double> tuple)
+ {
+ MutableDouble currentSma = currentSMAMap.get(tuple.getKey());
+ if (currentSma == null) { // first sma for the given key
+ double sma = tuple.getValue();
+ currentSMAMap.put(tuple.getKey(), new MutableDouble(sma));
+ //lastSMAMap.put(tuple.getKey(), new MutableDouble(sma));
+ } else { // move the current SMA value to the last SMA Map
+ //lastSMAMap.get(tuple.getKey()).setValue(currentSma.getValue());
+ currentSma.setValue(tuple.getValue()); // update the current SMA value
+ }
+ }
+
+ };
+ public final transient DefaultInputPort<KeyValPair<MerchantKey, Long>> txInputPort =
+ new DefaultInputPort<KeyValPair<MerchantKey, Long>>()
+ {
+ @Override
+ public void process(KeyValPair<MerchantKey, Long> tuple)
+ {
+ processTuple(tuple);
+ }
+
+ };
+
+ private void processTuple(KeyValPair<MerchantKey, Long> tuple)
+ {
+ MerchantKey merchantKey = tuple.getKey();
+ MutableDouble lastSma = lastSMAMap.get(tuple.getKey());
+ long txValue = tuple.getValue();
+ if (lastSma != null && txValue > lastSma.doubleValue()) {
+ double lastSmaValue = lastSma.doubleValue();
+ double change = txValue - lastSmaValue;
+ if (change > threshold) { // generate an alert
+ AverageAlertData data = getOutputData(merchantKey, txValue, change, lastSmaValue);
+ alerts.add(data);
+ //if (userGenerated) { // if its user generated only the pass it to WebSocket
+ if (merchantKey.merchantType == MerchantTransaction.MerchantType.BRICK_AND_MORTAR) {
+ avgAlertNotificationPort.emit(getOutputData(data, String.format(brickMortarAlertMsg, txValue, change, lastSmaValue, merchantKey.merchantId, merchantKey.terminalId)));
+ } else { // its internet based
+ avgAlertNotificationPort.emit(getOutputData(data, String.format(internetAlertMsg, txValue, change, lastSmaValue, merchantKey.merchantId)));
+
+ }
+ //}
+ }
+ }
+ }
+
+ @Override
+ public void endWindow()
+ {
+ for (AverageAlertData data : alerts) {
+ try {
+ avgAlertOutputPort.emit(JsonUtils.toJson(data));
+ } catch (IOException e) {
+ logger.warn("Exception while converting object to JSON", e);
+ }
+ }
+
+ alerts.clear();
+
+ for (Map.Entry<MerchantKey, MutableDouble> entry : currentSMAMap.entrySet()) {
+ MerchantKey key = entry.getKey();
+ MutableDouble currentSma = entry.getValue();
+ MutableDouble lastSma = lastSMAMap.get(key);
+ if (lastSma == null) {
+ lastSma = new MutableDouble(currentSma.doubleValue());
+ lastSMAMap.put(key, lastSma);
+ } else {
+ lastSma.setValue(currentSma.getValue());
+ }
+ }
+ }
+
+ private AverageAlertData getOutputData(MerchantKey key, long amount, double change, double lastSmaValue)
+ {
+ AverageAlertData data = new AverageAlertData();
+
+ data.merchantId = key.merchantId;
+ data.terminalId = key.terminalId == null ? 0 : key.terminalId;
+ data.zipCode = key.zipCode;
+ data.merchantType = key.merchantType;
+ data.amount = amount;
+ data.lastSmaValue = lastSmaValue;
+ data.change = change;
+ //data.userGenerated = userGenerated;
+ data.userGenerated = key.userGenerated;
+ data.time = System.currentTimeMillis();
+
+ return data;
+ }
+
+ private Map<String, Object> getOutputData(AverageAlertData data, String msg)
+ {
+ Map<String, Object> output = new HashMap<String, Object>();
+ output.put("message", msg);
+ output.put("alertType", "aboveAvg");
+ output.put("userGenerated", "" + data.userGenerated);
+ output.put("alertData", data);
+
+ try {
+ String str = mapper.writeValueAsString(output);
+ logger.debug("user generated tx alert: " + str);
+ } catch (Exception exc) {
+ //ignore
+ }
+ return output;
+ }
+
+ public int getThreshold()
+ {
+ return threshold;
+ }
+
+ public void setThreshold(int threshold)
+ {
+ this.threshold = threshold;
+ }
+
+ private static final Logger logger = LoggerFactory.getLogger(AverageAlertingOperator.class);
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/BankIdNumberAlertData.java
----------------------------------------------------------------------
diff --git a/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/BankIdNumberAlertData.java b/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/BankIdNumberAlertData.java
new file mode 100644
index 0000000..28cd19a
--- /dev/null
+++ b/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/BankIdNumberAlertData.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.frauddetect;
+
+/**
+ * POJO to capture data related to alerts for repetitive bank id number data usage.
+ *
+ * @since 0.9.0
+ */
+public class BankIdNumberAlertData
+{
+ public String merchantId;
+ public int terminalId;
+ public int zipCode;
+ public MerchantTransaction.MerchantType merchantType;
+ public String bankIdNum;
+ public int count;
+ public boolean userGenerated;
+ public long time;
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/BankIdNumberKey.java
----------------------------------------------------------------------
diff --git a/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/BankIdNumberKey.java b/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/BankIdNumberKey.java
new file mode 100644
index 0000000..eea6b14
--- /dev/null
+++ b/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/BankIdNumberKey.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.frauddetect;
+
+import java.io.Serializable;
+
+import com.datatorrent.lib.util.TimeBucketKey;
+
+/**
+ * Bank Id Number Key
+ *
+ * @since 0.9.0
+ */
+public class BankIdNumberKey extends TimeBucketKey implements Serializable
+{
+ public String bankIdNum;
+
+ public BankIdNumberKey()
+ {
+ }
+
+ @Override
+ public int hashCode()
+ {
+ int key = 0;
+ key |= (1 << 1);
+ key |= (bankIdNum.hashCode());
+ return super.hashCode() ^ key;
+ }
+
+ @Override
+ public boolean equals(Object obj)
+ {
+ if (!(obj instanceof BankIdNumberKey)) {
+ return false;
+ }
+ return super.equals(obj)
+ && bankIdNum.equals(((BankIdNumberKey)obj).bankIdNum);
+ }
+
+ @Override
+ public String toString()
+ {
+ StringBuilder sb = new StringBuilder(super.toString());
+ sb.append("|1:").append(bankIdNum);
+ return sb.toString();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/BankIdNumberSamplerOperator.java
----------------------------------------------------------------------
diff --git a/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/BankIdNumberSamplerOperator.java b/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/BankIdNumberSamplerOperator.java
new file mode 100644
index 0000000..0731e3c
--- /dev/null
+++ b/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/BankIdNumberSamplerOperator.java
@@ -0,0 +1,218 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.frauddetect;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.codehaus.jackson.JsonFactory;
+import org.codehaus.jackson.map.ObjectMapper;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.apex.examples.frauddetect.util.JsonUtils;
+
+import org.apache.commons.lang.mutable.MutableLong;
+
+import com.datatorrent.api.DefaultInputPort;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.common.util.BaseOperator;
+
+import com.datatorrent.lib.util.KeyValPair;
+
+/**
+ * Count the transactions for the underlying aggregation window if the same BIN is
+ * being used for more than defined number of transactions. Output the data as needed
+ * by Mongo output operator
+ *
+ * @since 0.9.0
+ */
+public class BankIdNumberSamplerOperator extends BaseOperator
+{
+ private final transient JsonFactory jsonFactory = new JsonFactory();
+ private final transient ObjectMapper mapper = new ObjectMapper(jsonFactory);
+ private int threshold;
+ private Map<MerchantKey, Map<String, BankIdNumData>> bankIdNumCountMap = new HashMap<MerchantKey, Map<String, BankIdNumData>>();
+ private static final String ALERT_MSG =
+ "Potential fraudulent CC transactions (same bank id %s and merchant %s) total transactions: %d";
+ /**
+ * Output the key-value pair for the BIN as key with the count as value.
+ */
+ public final transient DefaultOutputPort<String> countAlertOutputPort =
+ new DefaultOutputPort<String>();
+ public final transient DefaultOutputPort<Map<String, Object>> countAlertNotificationPort =
+ new DefaultOutputPort<Map<String, Object>>();
+
+ public int getThreshold()
+ {
+ return threshold;
+ }
+
+ public void setThreshold(int threshold)
+ {
+ this.threshold = threshold;
+ }
+
+ /*
+ public final transient DefaultInputPort<KeyValPair<MerchantKey, String>> txInputPort =
+ new DefaultInputPort<KeyValPair<MerchantKey, String>>()
+ {
+ @Override
+ public void process(KeyValPair<MerchantKey, String> tuple)
+ {
+ processTuple(tuple);
+ }
+
+ };
+
+ private void processTuple(KeyValPair<MerchantKey, String> tuple)
+ {
+ MerchantKey key = tuple.getKey();
+ Map<String, BankIdNumData> map = bankIdNumCountMap.get(key);
+ if (map == null) {
+ map = new HashMap<String, BankIdNumData>();
+ bankIdNumCountMap.put(key, map);
+ }
+ String bankIdNum = tuple.getValue();
+ BankIdNumData bankIdNumData = map.get(bankIdNum);
+ if (bankIdNumData == null) {
+ bankIdNumData = new BankIdNumData();
+ bankIdNumData.bankIdNum = bankIdNum;
+ map.put(bankIdNum, bankIdNumData);
+ }
+ bankIdNumData.count.increment();
+ if (key.userGenerated) {
+ bankIdNumData.userGenerated = true;
+ }
+ }
+ */
+
+ public final transient DefaultInputPort<KeyValPair<KeyValPair<MerchantKey, String>, Integer>> txCountInputPort =
+ new DefaultInputPort<KeyValPair<KeyValPair<MerchantKey, String>, Integer>>()
+ {
+ @Override
+ public void process(KeyValPair<KeyValPair<MerchantKey, String>, Integer> tuple)
+ {
+ processTuple(tuple.getKey(), tuple.getValue());
+ }
+
+ };
+
+ private void processTuple(KeyValPair<MerchantKey, String> tuple, Integer count)
+ {
+ MerchantKey key = tuple.getKey();
+ Map<String, BankIdNumData> map = bankIdNumCountMap.get(key);
+ if (map == null) {
+ map = new HashMap<String, BankIdNumData>();
+ bankIdNumCountMap.put(key, map);
+ }
+ String bankIdNum = tuple.getValue();
+ BankIdNumData bankIdNumData = map.get(bankIdNum);
+ if (bankIdNumData == null) {
+ bankIdNumData = new BankIdNumData();
+ bankIdNumData.bankIdNum = bankIdNum;
+ map.put(bankIdNum, bankIdNumData);
+ }
+ bankIdNumData.count.setValue(count);
+ if (key.userGenerated) {
+ bankIdNumData.userGenerated = true;
+ }
+ }
+
+ /**
+ * Go through the BIN Counter map and check if any of the values for the BIN exceed the threshold.
+ * If yes, generate the alert on the output port.
+ */
+ @Override
+ public void endWindow()
+ {
+ for (Map.Entry<MerchantKey, Map<String, BankIdNumData>> entry : bankIdNumCountMap.entrySet()) {
+ List<BankIdNumData> list = null;
+ MerchantKey key = entry.getKey();
+ if (key.merchantType == MerchantTransaction.MerchantType.INTERNET) {
+ continue;
+ }
+ list = dataOutput(entry.getValue());
+ if (list.size() > 0) {
+ for (BankIdNumData binData : list) {
+ BankIdNumberAlertData data = new BankIdNumberAlertData();
+ data.merchantId = key.merchantId;
+ data.terminalId = key.terminalId == null ? 0 : key.terminalId;
+ data.zipCode = key.zipCode;
+ data.merchantType = key.merchantType;
+ data.bankIdNum = binData.bankIdNum;
+ data.count = binData.count.intValue();
+ data.userGenerated = binData.userGenerated;
+ data.time = System.currentTimeMillis();
+ try {
+ countAlertOutputPort.emit(JsonUtils.toJson(data));
+ countAlertNotificationPort.emit(getOutputData(data));
+ } catch (IOException e) {
+ logger.warn("Exception while converting object to JSON: ", e);
+ }
+ }
+ }
+ }
+ bankIdNumCountMap.clear();
+ }
+
+ private List<BankIdNumData> dataOutput(Map<String, BankIdNumData> map)
+ {
+ List<BankIdNumData> list = new ArrayList<BankIdNumData>();
+ int count = 0;
+ for (Map.Entry<String, BankIdNumData> bankIdEntry : map.entrySet()) {
+ BankIdNumData data = bankIdEntry.getValue();
+ if (data.count.intValue() > threshold) {
+ list.add(data);
+ }
+ }
+ return list;
+ }
+
+ private Map<String, Object> getOutputData(BankIdNumberAlertData data)
+ {
+ Map<String, Object> output = new HashMap<String, Object>();
+ output.put("message", String.format(ALERT_MSG, data.bankIdNum, data.merchantId, data.count));
+ output.put("alertType", "sameBankId");
+ output.put("userGenerated", "" + data.userGenerated);
+ output.put("alertData", data);
+
+ try {
+ String str = mapper.writeValueAsString(output);
+ logger.debug("user generated tx alert: " + str);
+ } catch (Exception exc) {
+ //ignore
+ }
+
+ return output;
+ }
+
+ public static final class BankIdNumData
+ {
+ public String bankIdNum;
+ public MutableLong count = new MutableLong();
+ public boolean userGenerated = false;
+ }
+
+ private static final Logger logger = LoggerFactory.getLogger(BankIdNumberSamplerOperator.class);
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/CreditCardAlertData.java
----------------------------------------------------------------------
diff --git a/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/CreditCardAlertData.java b/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/CreditCardAlertData.java
new file mode 100644
index 0000000..f46f84a
--- /dev/null
+++ b/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/CreditCardAlertData.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.frauddetect;
+
+/**
+ * POJO to capture data related to alerts for credit card number.
+ *
+ * @since 0.9.0
+ */
+public class CreditCardAlertData
+{
+ public String merchantId;
+ public int terminalId;
+ public int zipCode;
+ public MerchantTransaction.MerchantType merchantType;
+ public String fullCcNum;
+ public long small;
+ public long large;
+ public double threshold;
+ public boolean userGenerated;
+ public long time;
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/CreditCardAmountSamplerOperator.java
----------------------------------------------------------------------
diff --git a/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/CreditCardAmountSamplerOperator.java b/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/CreditCardAmountSamplerOperator.java
new file mode 100644
index 0000000..235e36e
--- /dev/null
+++ b/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/CreditCardAmountSamplerOperator.java
@@ -0,0 +1,206 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.frauddetect;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.codehaus.jackson.JsonFactory;
+import org.codehaus.jackson.map.ObjectMapper;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.apex.examples.frauddetect.util.JsonUtils;
+import org.apache.commons.lang.mutable.MutableLong;
+
+import com.datatorrent.api.DefaultInputPort;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.common.util.BaseOperator;
+import com.datatorrent.lib.util.KeyValPair;
+
+/**
+ * An operator to alert in case a transaction of a small lowAmount is followed by a transaction which is significantly larger for a given credit card number.
+ * This is done for each transaction. This also means that this happens for each individual credit card.
+ * It accepts merchant transaction object and for each CC listed in the transaction(s), checks for the transaction amounts. An alert is raised if the transaction
+ * lowAmount is significantly > the lowest amt in this window.
+ *
+ * @since 0.9.0
+ */
+public class CreditCardAmountSamplerOperator extends BaseOperator
+{
+ private final transient JsonFactory jsonFactory = new JsonFactory();
+ private final transient ObjectMapper mapper = new ObjectMapper(jsonFactory);
+ private static final Logger logger = LoggerFactory.getLogger(Application.class);
+ // Factor to be applied to existing lowAmount to flag potential alerts.
+ private double threshold = 9500;
+ private Map<String, CreditCardInfo> ccTxnMap = new HashMap<String, CreditCardInfo>();
+ //private Map<String, MutableLong> ccQueryTxnMap = new HashMap<String, MutableLong>();
+ private List<CreditCardAlertData> alerts = new ArrayList<CreditCardAlertData>();
+ //private List<CreditCardAlertData> userAlerts = new ArrayList<CreditCardAlertData>();
+ private static final String ALERT_MSG =
+ "Potential fraudulent CC transactions (small one USD %d followed by large USD %d) performed using credit card: %s";
+ public final transient DefaultOutputPort<String> ccAlertOutputPort = new DefaultOutputPort<String>();
+ /*
+ public final transient DefaultOutputPort<Map<String, Object>> ccUserAlertOutputPort = new DefaultOutputPort<Map<String, Object>>();
+ */
+ public final transient DefaultOutputPort<Map<String, Object>> ccAlertNotificationPort = new DefaultOutputPort<Map<String, Object>>();
+
+ public double getThreshold()
+ {
+ return threshold;
+ }
+
+ public void setThreshold(double threshold)
+ {
+ this.threshold = threshold;
+ }
+
+ private void processTuple(KeyValPair<MerchantKey, CreditCardData> tuple, Map<String, CreditCardInfo> txMap)
+ {
+ String fullCcNum = tuple.getValue().fullCcNum;
+ long ccAmount = tuple.getValue().amount;
+ MerchantKey key = tuple.getKey();
+
+ CreditCardInfo cardInfo = txMap.get(fullCcNum);
+
+ if (cardInfo != null) {
+ long currentSmallValue = cardInfo.lowAmount.longValue();
+ if (ccAmount < currentSmallValue) {
+ cardInfo.lowAmount.setValue(ccAmount);
+ cardInfo.time = key.time;
+ } else if (ccAmount > (currentSmallValue + threshold)) {
+ // If the transaction lowAmount is > 70% of the min. lowAmount, send an alert.
+
+ CreditCardAlertData data = new CreditCardAlertData();
+
+ data.merchantId = key.merchantId;
+ data.terminalId = key.terminalId == null ? 0 : key.terminalId;
+ data.zipCode = key.zipCode;
+ data.merchantType = key.merchantType;
+ data.fullCcNum = fullCcNum;
+ data.small = currentSmallValue;
+ data.large = ccAmount;
+ data.threshold = threshold;
+ data.userGenerated = key.userGenerated;
+ data.time = System.currentTimeMillis();
+
+ alerts.add(data);
+
+ /*
+ if (userGenerated){
+ userAlerts.add(data);
+ }
+ */
+ ccAlertNotificationPort.emit(getOutputData(data));
+
+ // Any high value transaction after a low value transaction with difference greater than threshold
+ // will trigger the alert. Not resetting the low value also helps in a system generated transaction
+ // alert not resetting the low value from a user generated transaction
+ //txMap.remove(fullCcNum);
+ }
+ } else {
+ cardInfo = new CreditCardInfo();
+ cardInfo.lowAmount.setValue(ccAmount);
+ cardInfo.time = key.time;
+ txMap.put(fullCcNum, cardInfo);
+ }
+ }
+
+ public transient DefaultInputPort<KeyValPair<MerchantKey, CreditCardData>> inputPort =
+ new DefaultInputPort<KeyValPair<MerchantKey, CreditCardData>>()
+ {
+ //
+ // This function checks if a CC entry exists.
+ // If so, it checks whether the current transaction is for an lowAmount lesser than the one stored in the hashmap. If so, this becomes the min. transaction lowAmount.
+ // If the lowAmount is > 70% of the existing lowAmount in the hash map, raise an alert.
+ //
+ @Override
+ public void process(KeyValPair<MerchantKey, CreditCardData> tuple)
+ {
+
+ processTuple(tuple, ccTxnMap);
+
+ }
+
+ };
+
+ @Override
+ public void endWindow()
+ {
+
+ for (CreditCardAlertData data : alerts) {
+ try {
+ ccAlertOutputPort.emit(JsonUtils.toJson(data));
+ } catch (IOException e) {
+ logger.warn("Exception while converting object to JSON", e);
+ }
+ }
+
+ //for (CreditCardAlertData data: userAlerts) {
+ /*for (CreditCardAlertData data: alerts) {
+ ccAlertNotificationPort.emit(getOutputData(data));
+ }*/
+
+ long ctime = System.currentTimeMillis();
+ Iterator<Map.Entry<String, CreditCardInfo>> iterator = ccTxnMap.entrySet().iterator();
+ while (iterator.hasNext()) {
+ Map.Entry<String, CreditCardInfo> entry = iterator.next();
+ long time = entry.getValue().time;
+ if ((ctime - time) > 60000) {
+ iterator.remove();
+ }
+ }
+
+ //ccTxnMap.clear();
+ alerts.clear();
+
+ //ccQueryTxnMap.clear();
+ //userAlerts.clear();
+ }
+
+ private static class CreditCardInfo
+ {
+ MutableLong lowAmount = new MutableLong();
+ Long time;
+ }
+
+ private Map<String, Object> getOutputData(CreditCardAlertData data)
+ {
+ Map<String, Object> output = new HashMap<String, Object>();
+ output.put("message", String.format(ALERT_MSG, data.small, data.large, data.fullCcNum));
+ output.put("alertType", "smallThenLarge");
+ output.put("userGenerated", "" + data.userGenerated);
+ output.put("alertData", data);
+
+ try {
+ String str = mapper.writeValueAsString(output);
+ logger.debug("Alert generated: " + str + " userGenerated: " + data.userGenerated);
+ } catch (Exception exc) {
+ //ignore
+ }
+
+ return output;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/CreditCardData.java
----------------------------------------------------------------------
diff --git a/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/CreditCardData.java b/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/CreditCardData.java
new file mode 100644
index 0000000..7c667d6
--- /dev/null
+++ b/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/CreditCardData.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.frauddetect;
+
+/**
+ * Credit Card Data
+ *
+ * @since 0.9.0
+ */
+public class CreditCardData
+{
+ public String fullCcNum;
+ public long amount;
+
+ public CreditCardData()
+ {
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/MerchantKey.java
----------------------------------------------------------------------
diff --git a/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/MerchantKey.java b/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/MerchantKey.java
new file mode 100644
index 0000000..d73c693
--- /dev/null
+++ b/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/MerchantKey.java
@@ -0,0 +1,128 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.frauddetect;
+
+
+import java.io.Serializable;
+
+/**
+ * A time-based key for merchant data.
+ *
+ * @since 0.9.0
+ */
+public class MerchantKey implements Serializable
+{
+ public String merchantId;
+ public Integer terminalId;
+ public Integer zipCode;
+ public String country;
+ public MerchantTransaction.MerchantType merchantType;
+ public Long time;
+ public boolean userGenerated;
+
+ public MerchantKey()
+ {
+ }
+
+ @Override
+ public int hashCode()
+ {
+ int key = 0;
+ if (merchantId != null) {
+ key |= (1 << 1);
+ key |= (merchantId.hashCode());
+ }
+ if (terminalId != null) {
+ key |= (1 << 2);
+ key |= (terminalId << 2);
+ }
+ if (zipCode != null) {
+ key |= (1 << 3);
+ key |= (zipCode << 3);
+ }
+ if (country != null) {
+ key |= (1 << 4);
+ key |= (country.hashCode());
+ }
+ if (merchantType != null) {
+ key |= (1 << 5);
+ key |= (merchantType.hashCode());
+ }
+ return key;
+ }
+
+ @Override
+ public boolean equals(Object obj)
+ {
+ if (!(obj instanceof MerchantKey)) {
+ return false;
+ }
+ MerchantKey mkey = (MerchantKey)obj;
+ return checkStringEqual(this.merchantId, mkey.merchantId)
+ && checkIntEqual(this.terminalId, mkey.terminalId)
+ && checkIntEqual(this.zipCode, mkey.zipCode)
+ && checkStringEqual(this.country, mkey.country)
+ && checkIntEqual(this.merchantType.ordinal(), mkey.merchantType.ordinal());
+ }
+
+ private boolean checkIntEqual(Integer a, Integer b)
+ {
+ if ((a == null) && (b == null)) {
+ return true;
+ }
+ if ((a != null) && (b != null) && a.intValue() == b.intValue()) {
+ return true;
+ }
+ return false;
+ }
+
+ private boolean checkStringEqual(String a, String b)
+ {
+ if ((a == null) && (b == null)) {
+ return true;
+ }
+ if ((a != null) && a.equals(b)) {
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public String toString()
+ {
+ StringBuilder sb = new StringBuilder();
+ if (merchantId != null) {
+ sb.append("|1:").append(merchantId);
+ }
+ if (terminalId != null) {
+ sb.append("|2:").append(terminalId);
+ }
+ if (zipCode != null) {
+ sb.append("|3:").append(zipCode);
+ }
+ if (country != null) {
+ sb.append("|4:").append(country);
+ }
+ if (merchantType != null) {
+ sb.append("|5:").append(merchantType);
+ }
+ return sb.toString();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/MerchantQueryInputHandler.java
----------------------------------------------------------------------
diff --git a/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/MerchantQueryInputHandler.java b/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/MerchantQueryInputHandler.java
new file mode 100644
index 0000000..e6a4680
--- /dev/null
+++ b/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/MerchantQueryInputHandler.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.frauddetect;
+
+import java.util.Map;
+
+/**
+ * Common utility class that can be used by all other operators to handle user input
+ * captured from the Web socket input port.
+ *
+ * @since 0.9.0
+ */
+public class MerchantQueryInputHandler
+{
+ public static final String KEY_DATA = "data";
+ public static final String KEY_MERCHANT_ID = "merchantId";
+ public static final String KEY_TERMINAL_ID = "terminalId";
+ public static final String KEY_ZIP_CODE = "zipCode";
+
+ public static MerchantKey process(Map<String, Object> tuple)
+ {
+ String merchantId = null;
+ Integer terminalId = null;
+ Integer zipCode = null;
+
+ // ignoring other top-level attributes.
+ Map<String, Object> data = (Map<String, Object>)tuple.get(KEY_DATA);
+ if (data.get(KEY_MERCHANT_ID) != null) {
+ merchantId = (String)data.get(KEY_MERCHANT_ID);
+ }
+ if (data.get(KEY_TERMINAL_ID) != null) {
+ terminalId = (Integer)data.get(KEY_TERMINAL_ID);
+ }
+ if (data.get(KEY_ZIP_CODE) != null) {
+ zipCode = (Integer)data.get(KEY_ZIP_CODE);
+ }
+
+ MerchantKey key = new MerchantKey();
+ key.merchantId = merchantId;
+ key.terminalId = terminalId;
+ key.zipCode = zipCode;
+ key.country = "USA";
+ if (merchantId != null) {
+ key.merchantType = key.merchantId.equalsIgnoreCase(MerchantTransactionGenerator.merchantIds[2])
+ || key.merchantId.equalsIgnoreCase(MerchantTransactionGenerator.merchantIds[3])
+ ? MerchantTransaction.MerchantType.INTERNET
+ : MerchantTransaction.MerchantType.BRICK_AND_MORTAR;
+ }
+ return key;
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/MerchantTransaction.java
----------------------------------------------------------------------
diff --git a/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/MerchantTransaction.java b/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/MerchantTransaction.java
new file mode 100644
index 0000000..a722492
--- /dev/null
+++ b/examples/frauddetect/src/main/java/org/apache/apex/examples/frauddetect/MerchantTransaction.java
@@ -0,0 +1,202 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.frauddetect;
+
+import java.io.Serializable;
+
+/**
+ * POJO for BIN Alert related data.
+ *
+ * @since 0.9.0
+ */
+public class MerchantTransaction implements Serializable
+{
+ public enum MerchantType
+ {
+ UNDEFINED, BRICK_AND_MORTAR, INTERNET
+ }
+
+ public enum TransactionType
+ {
+ UNDEFINED, POS
+ }
+
+ public String ccNum;
+ public String bankIdNum;
+ public String fullCcNum;
+ public Long amount;
+ public String merchantId;
+ public Integer terminalId;
+ public Integer zipCode;
+ public String country;
+ public MerchantType merchantType = MerchantType.UNDEFINED;
+ public TransactionType transactionType = TransactionType.UNDEFINED;
+ public Long time;
+ public boolean userGenerated;
+
+ public MerchantTransaction()
+ {
+ }
+
+ @Override
+ public int hashCode()
+ {
+ int key = 0;
+ if (ccNum != null) {
+ key |= (1 << 1);
+ key |= (ccNum.hashCode());
+ }
+ if (bankIdNum != null) {
+ key |= (1 << 2);
+ key |= (bankIdNum.hashCode());
+ }
+ if (amount != null) {
+ key |= (1 << 6);
+ key |= (amount << 4);
+ }
+ if (merchantId != null) {
+ key |= (1 << 3);
+ key |= (merchantId.hashCode());
+ }
+ if (terminalId != null) {
+ key |= (1 << 4);
+ key |= (terminalId << 2);
+ }
+ if (zipCode != null) {
+ key |= (1 << 5);
+ key |= (zipCode << 3);
+ }
+ if (country != null) {
+ key |= (1 << 7);
+ key |= (country.hashCode());
+ }
+ if (merchantType != null) {
+ key |= (1 << 8);
+ key |= (merchantType.hashCode());
+ }
+ if (transactionType != null) {
+ key |= (1 << 9);
+ key |= (transactionType.hashCode());
+ }
+ if (fullCcNum != null) {
+ key |= (1 << 10);
+ key |= (fullCcNum.hashCode());
+ }
+ if (time != null) {
+ key |= (1 << 11);
+ key |= (time << 2);
+ }
+
+ return key;
+ }
+
+ @Override
+ public boolean equals(Object obj)
+ {
+ if (!(obj instanceof MerchantTransaction)) {
+ return false;
+ }
+ MerchantTransaction mtx = (MerchantTransaction)obj;
+ return checkStringEqual(this.ccNum, mtx.ccNum)
+ && checkStringEqual(this.bankIdNum, mtx.bankIdNum)
+ && checkLongEqual(this.amount, mtx.amount)
+ && checkStringEqual(this.merchantId, mtx.merchantId)
+ && checkIntEqual(this.terminalId, mtx.terminalId)
+ && checkIntEqual(this.zipCode, mtx.zipCode)
+ && checkStringEqual(this.country, mtx.country)
+ && checkIntEqual(this.merchantType.ordinal(), mtx.merchantType.ordinal())
+ && checkIntEqual(this.transactionType.ordinal(), mtx.transactionType.ordinal())
+ && checkStringEqual(this.fullCcNum, mtx.fullCcNum)
+ && checkLongEqual(this.time, mtx.time);
+ }
+
+ private boolean checkIntEqual(Integer a, Integer b)
+ {
+ if ((a == null) && (b == null)) {
+ return true;
+ }
+ if ((a != null) && (b != null) && a.intValue() == b.intValue()) {
+ return true;
+ }
+ return false;
+ }
+
+ private boolean checkLongEqual(Long a, Long b)
+ {
+ if ((a == null) && (b == null)) {
+ return true;
+ }
+ if ((a != null) && (b != null) && a.longValue() == b.longValue()) {
+ return true;
+ }
+ return false;
+ }
+
+ private boolean checkStringEqual(String a, String b)
+ {
+ if ((a == null) && (b == null)) {
+ return true;
+ }
+ if ((a != null) && a.equals(b)) {
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public String toString()
+ {
+ StringBuilder sb = new StringBuilder();
+ if (ccNum != null) {
+ sb.append("|0:").append(ccNum);
+ }
+ if (bankIdNum != null) {
+ sb.append("|1:").append(bankIdNum);
+ }
+ if (fullCcNum != null) {
+ sb.append("|2:").append(fullCcNum);
+ }
+ if (amount != null) {
+ sb.append("|3:").append(amount);
+ }
+ if (merchantId != null) {
+ sb.append("|4:").append(merchantId);
+ }
+ if (terminalId != null) {
+ sb.append("|5:").append(terminalId);
+ }
+ if (zipCode != null) {
+ sb.append("|6:").append(zipCode);
+ }
+ if (country != null) {
+ sb.append("|7:").append(country);
+ }
+ if (merchantType != null) {
+ sb.append("|8:").append(merchantType);
+ }
+ if (transactionType != null) {
+ sb.append("|9:").append(transactionType);
+ }
+ if (time != null) {
+ sb.append("|10:").append(time);
+ }
+ return sb.toString();
+ }
+
+}
[19/30] apex-malhar git commit: Renamed demos to examples. Packages
and artifactid names are changed as suggested.
Posted by th...@apache.org.
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/pom.xml
----------------------------------------------------------------------
diff --git a/demos/pom.xml b/demos/pom.xml
deleted file mode 100644
index 99cfcd2..0000000
--- a/demos/pom.xml
+++ /dev/null
@@ -1,231 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.apex</groupId>
- <artifactId>malhar</artifactId>
- <version>3.7.0-SNAPSHOT</version>
- </parent>
-
- <artifactId>malhar-demos</artifactId>
- <packaging>pom</packaging>
- <name>Apache Apex Malhar Demos</name>
-
- <properties>
- <apex.apppackage.groupid>${project.groupId}</apex.apppackage.groupid>
- <apex.apppackage.classpath>lib/*.jar</apex.apppackage.classpath>
- <semver.plugin.skip>true</semver.plugin.skip>
- <maven.deploy.skip>true</maven.deploy.skip>
- </properties>
-
- <profiles>
- <profile>
- <id>demo-plugin-activation</id>
- <activation>
- <file>
- <exists>${basedir}/src/main</exists>
- </file>
- </activation>
- <build>
- <plugins>
- <plugin>
- <artifactId>maven-eclipse-plugin</artifactId>
- <version>2.9</version>
- <configuration>
- <downloadSources>true</downloadSources>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-compiler-plugin</artifactId>
- <version>2.3.2</version>
- <configuration>
- <encoding>UTF-8</encoding>
- <source>1.7</source>
- <target>1.7</target>
- <debug>true</debug>
- <optimize>false</optimize>
- <showDeprecation>true</showDeprecation>
- <showWarnings>true</showWarnings>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-dependency-plugin</artifactId>
- <version>2.8</version>
- <executions>
- <execution>
- <id>copy-dependencies</id>
- <phase>prepare-package</phase>
- <goals>
- <goal>copy-dependencies</goal>
- </goals>
- <configuration>
- <outputDirectory>target/deps</outputDirectory>
- <includeScope>runtime</includeScope>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <executions>
- <execution>
- <id>app-package-assembly</id>
- <phase>package</phase>
- <goals>
- <goal>single</goal>
- </goals>
- <configuration>
- <finalName>${project.artifactId}-${project.version}-apexapp</finalName>
- <appendAssemblyId>false</appendAssemblyId>
- <descriptors>
- <descriptor>src/assemble/appPackage.xml</descriptor>
- </descriptors>
- <archiverConfig>
- <defaultDirectoryMode>0755</defaultDirectoryMode>
- </archiverConfig>
- <archive>
- <manifestEntries>
- <Class-Path>${apex.apppackage.classpath}</Class-Path>
- <DT-Engine-Version>${apex.core.version}</DT-Engine-Version>
- <DT-App-Package-Group-Id>${apex.apppackage.groupid}</DT-App-Package-Group-Id>
- <DT-App-Package-Name>${project.artifactId}</DT-App-Package-Name>
- <DT-App-Package-Version>${project.version}</DT-App-Package-Version>
- <DT-App-Package-Display-Name>${project.name}</DT-App-Package-Display-Name>
- <DT-App-Package-Description>${project.description}</DT-App-Package-Description>
- </manifestEntries>
- </archive>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <artifactId>maven-antrun-plugin</artifactId>
- <version>1.7</version>
- <executions>
- <execution>
- <phase>package</phase>
- <configuration>
- <target>
- <move file="${project.build.directory}/${project.artifactId}-${project.version}-apexapp.jar"
- tofile="${project.build.directory}/${project.artifactId}-${project.version}.apa" />
- </target>
- </configuration>
- <goals>
- <goal>run</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <version>1.9.1</version>
- <executions>
- <execution>
- <id>attach-artifacts</id>
- <phase>package</phase>
- <goals>
- <goal>attach-artifact</goal>
- </goals>
- <configuration>
- <artifacts>
- <artifact>
- <file>target/${project.artifactId}-${project.version}.apa</file>
- <type>apa</type>
- </artifact>
- </artifacts>
- <skipAttach>false</skipAttach>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- </profile>
- <profile>
- <id>all-modules</id>
- <modules>
- <module>distributedistinct</module>
- <module>highlevelapi</module>
- <module>sql</module>
- </modules>
- </profile>
- </profiles>
-
- <modules>
- <module>machinedata</module>
- <module>pi</module>
- <module>twitter</module>
- <module>yahoofinance</module>
- <module>frauddetect</module>
- <module>mobile</module>
- <module>wordcount</module>
- <module>mrmonitor</module>
- <module>mroperator</module>
- <module>uniquecount</module>
- <module>r</module>
- <module>echoserver</module>
- <module>iteration</module>
- </modules>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.apex</groupId>
- <artifactId>apex-common</artifactId>
- <version>${apex.core.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.10</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.apex</groupId>
- <artifactId>malhar-library</artifactId>
- <version>${project.version}</version>
- <exclusions>
- <exclusion>
- <groupId>*</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.apex</groupId>
- <artifactId>malhar-library</artifactId>
- <version>${project.version}</version>
- <classifier>tests</classifier>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>*</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- </dependencies>
-
-</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/r/pom.xml
----------------------------------------------------------------------
diff --git a/demos/r/pom.xml b/demos/r/pom.xml
deleted file mode 100644
index d8b73f8..0000000
--- a/demos/r/pom.xml
+++ /dev/null
@@ -1,83 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>r-demo</artifactId>
- <packaging>jar</packaging>
-
- <name>Apache Apex Malhar R Demo</name>
- <description>Apex demo applications for using R.</description>
-
- <parent>
- <groupId>org.apache.apex</groupId>
- <artifactId>malhar-demos</artifactId>
- <version>3.7.0-SNAPSHOT</version>
- </parent>
-
- <properties>
- <skipTests>true</skipTests>
- </properties>
-
- <repositories>
- <repository>
- <id>datatorrent-3rd-party</id>
- <name>Embedded repository for dependencies not available online</name>
- <url>https://www.datatorrent.com/maven/content/repositories/thirdparty</url>
- <snapshots>
- <updatePolicy>daily</updatePolicy>
- </snapshots>
- <releases>
- <updatePolicy>daily</updatePolicy>
- </releases>
- </repository>
- </repositories>
-
- <dependencies>
- <dependency>
- <groupId>org.rosuda</groupId>
- <artifactId>jri</artifactId>
- <version>1.0</version>
- </dependency>
- <dependency>
- <groupId>org.rosuda</groupId>
- <artifactId>rengine</artifactId>
- <version>1.0</version>
- </dependency>
- <dependency>
- <groupId>org.rosuda</groupId>
- <artifactId>jriengine</artifactId>
- <version>1.0</version>
- </dependency>
- <dependency>
- <groupId>org.apache.apex</groupId>
- <artifactId>malhar-contrib</artifactId>
- <version>${project.version}</version>
- <exclusions>
- <exclusion>
- <groupId>*</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- </dependencies>
-</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/r/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/demos/r/src/assemble/appPackage.xml b/demos/r/src/assemble/appPackage.xml
deleted file mode 100644
index 4138cf2..0000000
--- a/demos/r/src/assemble/appPackage.xml
+++ /dev/null
@@ -1,59 +0,0 @@
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
- <id>appPackage</id>
- <formats>
- <format>jar</format>
- </formats>
- <includeBaseDirectory>false</includeBaseDirectory>
- <fileSets>
- <fileSet>
- <directory>${basedir}/target/</directory>
- <outputDirectory>/app</outputDirectory>
- <includes>
- <include>${project.artifactId}-${project.version}.jar</include>
- </includes>
- </fileSet>
- <fileSet>
- <directory>${basedir}/target/deps</directory>
- <outputDirectory>/lib</outputDirectory>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/site/conf</directory>
- <outputDirectory>/conf</outputDirectory>
- <includes>
- <include>*.xml</include>
- </includes>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/main/resources/META-INF</directory>
- <outputDirectory>/META-INF</outputDirectory>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/main/resources/app</directory>
- <outputDirectory>/app</outputDirectory>
- </fileSet>
- </fileSets>
-
-</assembly>
-
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/r/src/main/java/com/datatorrent/demos/r/oldfaithful/FaithfulKey.java
----------------------------------------------------------------------
diff --git a/demos/r/src/main/java/com/datatorrent/demos/r/oldfaithful/FaithfulKey.java b/demos/r/src/main/java/com/datatorrent/demos/r/oldfaithful/FaithfulKey.java
deleted file mode 100755
index b2bfd46..0000000
--- a/demos/r/src/main/java/com/datatorrent/demos/r/oldfaithful/FaithfulKey.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.r.oldfaithful;
-
-/**
- * @since 2.1.0
- */
-public class FaithfulKey
-{
-
- private static final long serialVersionUID = 201403251620L;
-
- private double eruptionDuration;
- private int waitingTime;
-
- public FaithfulKey()
- {
- }
-
- public double getEruptionDuration()
- {
- return eruptionDuration;
- }
-
- public void setEruptionDuration(double eruptionDuration)
- {
- this.eruptionDuration = eruptionDuration;
- }
-
- public int getWaitingTime()
- {
- return waitingTime;
- }
-
- public void setWaitingTime(int waitingTime)
- {
- this.waitingTime = waitingTime;
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/r/src/main/java/com/datatorrent/demos/r/oldfaithful/FaithfulRScript.java
----------------------------------------------------------------------
diff --git a/demos/r/src/main/java/com/datatorrent/demos/r/oldfaithful/FaithfulRScript.java b/demos/r/src/main/java/com/datatorrent/demos/r/oldfaithful/FaithfulRScript.java
deleted file mode 100755
index cf49848..0000000
--- a/demos/r/src/main/java/com/datatorrent/demos/r/oldfaithful/FaithfulRScript.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.r.oldfaithful;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.datatorrent.api.Context;
-import com.datatorrent.api.DefaultInputPort;
-import com.datatorrent.api.annotation.InputPortFieldAnnotation;
-import com.datatorrent.contrib.r.RScript;
-
-/**
- * @since 2.1.0
- */
-public class FaithfulRScript extends RScript
-{
-
- private transient List<FaithfulKey> readingsList = new ArrayList<FaithfulKey>();
- private int elapsedTime;
- private static final Logger LOG = LoggerFactory.getLogger(FaithfulRScript.class);
-
- public FaithfulRScript()
- {
- super();
- }
-
- public FaithfulRScript(String rScriptFilePath, String rFunction, String returnVariable)
- {
- super(rScriptFilePath, rFunction, returnVariable);
- }
-
- @InputPortFieldAnnotation(optional = true)
- public final transient DefaultInputPort<FaithfulKey> faithfulInput = new DefaultInputPort<FaithfulKey>()
- {
- @Override
- public void process(FaithfulKey tuple)
- {
- // Create a map of ("String", values) to be passed to the process
- // function in the RScipt operator's process()
- readingsList.add(tuple);
-
- }
-
- };
-
- @InputPortFieldAnnotation(optional = true)
- public final transient DefaultInputPort<Integer> inputElapsedTime = new DefaultInputPort<Integer>()
- {
- @Override
- public void process(Integer eT)
- {
- elapsedTime = eT;
- }
- };
-
- @Override
- public void setup(Context.OperatorContext context)
- {
- super.setup(context);
- }
-
- @Override
- public void endWindow()
- {
- if (readingsList.size() == 0) {
- return;
- }
- LOG.info("Input data size: readingsList - " + readingsList.size());
-
- double[] eruptionDuration = new double[readingsList.size()];
- int[] waitingTime = new int[readingsList.size()];
-
- for (int i = 0; i < readingsList.size(); i++) {
- eruptionDuration[i] = readingsList.get(i).getEruptionDuration();
- waitingTime[i] = readingsList.get(i).getWaitingTime();
- }
- LOG.info("Input data size: eruptionDuration - " + eruptionDuration.length);
- LOG.info("Input data size: waitingTime - " + waitingTime.length);
-
- HashMap<String, Object> map = new HashMap<String, Object>();
-
- map.put("ELAPSEDTIME", elapsedTime);
- map.put("ERUPTIONS", eruptionDuration);
- map.put("WAITING", waitingTime);
-
- super.process(map);
- readingsList.clear();
- map.clear();
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/r/src/main/java/com/datatorrent/demos/r/oldfaithful/InputGenerator.java
----------------------------------------------------------------------
diff --git a/demos/r/src/main/java/com/datatorrent/demos/r/oldfaithful/InputGenerator.java b/demos/r/src/main/java/com/datatorrent/demos/r/oldfaithful/InputGenerator.java
deleted file mode 100755
index c45cd50..0000000
--- a/demos/r/src/main/java/com/datatorrent/demos/r/oldfaithful/InputGenerator.java
+++ /dev/null
@@ -1,145 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.r.oldfaithful;
-
-import java.util.Random;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.datatorrent.api.Context;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.api.InputOperator;
-
-/**
- * The InputGenerator operator is used to generate input for the 'Old Faithful Geyser" application.
- * This application accepts readings for the waiting time and the subsequent eruption duration
- * of the 'Old Faithful' and based on this data, tries to predict the eruption duration of the next
- * eruption given the elapsed time since the last eruption.
- * The training data is generated for an application window and consists of multiple
- * waiting times and eruption duration values.
- * For every application window, it generates only one 'elapsed time' input for which the
- * prediction would be made.
- *
- * @since 2.1.0
- */
-
-public class InputGenerator implements InputOperator
-{
-
- @SuppressWarnings("unused")
- private static final Logger LOG = LoggerFactory.getLogger(InputGenerator.class);
- private int blastCount = 1000;
- private Random random = new Random();
- private static int emitCount = 0;
-
- public final transient DefaultOutputPort<FaithfulKey> outputPort = new DefaultOutputPort<FaithfulKey>();
-
- public final transient DefaultOutputPort<Integer> elapsedTime = new DefaultOutputPort<Integer>();
-
- public void setBlastCount(int blastCount)
- {
- this.blastCount = blastCount;
- }
-
- @Override
- public void beginWindow(long windowId)
- {
- }
-
- @Override
- public void endWindow()
- {
- }
-
- @Override
- public void setup(Context.OperatorContext context)
- {
- }
-
- @Override
- public void teardown()
- {
- }
-
- private int nextRandomId(int min, int max)
- {
- int id;
- do {
- id = (int)Math.abs(Math.round(random.nextGaussian() * max));
- }
- while (id >= max);
-
- if (id < min) {
- id = min;
- }
- try {
- // Slowdown input generation
- if (emitCount++ % 97 == 0) {
- Thread.sleep(1);
- }
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- return id;
- }
-
- @Override
- public void emitTuples()
- {
- boolean elapsedTimeSent = false;
-
- try {
- for (int i = 0; i < blastCount; ++i) {
- int waitingTime = nextRandomId(3600, 36000);
-
- double eruptionDuration = -2.15 + 0.05 * waitingTime;
- emitTuple(eruptionDuration, waitingTime);
-
- if (!elapsedTimeSent) {
- int eT = 0;
-
- if (i % 100 == 0) {
- eT = 54 + waitingTime;
-
- emitElapsedTime(eT);
- elapsedTimeSent = true;
- }
- }
- }
- } catch (Exception ex) {
- throw new RuntimeException(ex);
- }
- }
-
- private void emitTuple(double eruptionDuration, int waitingTime)
- {
- FaithfulKey faithfulkey = new FaithfulKey();
-
- faithfulkey.setEruptionDuration(eruptionDuration);
- faithfulkey.setWaitingTime(waitingTime);
-
- this.outputPort.emit(faithfulkey);
- }
-
- private void emitElapsedTime(int eT)
- {
- this.elapsedTime.emit(eT);
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/r/src/main/java/com/datatorrent/demos/r/oldfaithful/OldFaithfulApplication.java
----------------------------------------------------------------------
diff --git a/demos/r/src/main/java/com/datatorrent/demos/r/oldfaithful/OldFaithfulApplication.java b/demos/r/src/main/java/com/datatorrent/demos/r/oldfaithful/OldFaithfulApplication.java
deleted file mode 100755
index 0483767..0000000
--- a/demos/r/src/main/java/com/datatorrent/demos/r/oldfaithful/OldFaithfulApplication.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.r.oldfaithful;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-import com.datatorrent.lib.io.ConsoleOutputOperator;
-
-/**
- * The application attempts to simulate 'Old Faithful Geyser" eruption.
- * This application accepts readings for the waiting time and the subsequent eruption duration
- * of the 'Old Faithful' and based on this data, tries to predict the eruption duration of the next
- * eruption given the elapsed time since the last eruption.
- * The training data is generated for an application window and consists of multiple
- * waiting times and eruption duration values.
- * For every application window, it generates only one 'elapsed time' input for which the
- * prediction would be made.
- * Model in R is in file ruptionModel.R located at
- * demos/r/src/main/resources/com/datatorrent/demos/oldfaithful/ directory
- *
- * @since 2.1.0
- */
-
-@ApplicationAnnotation(name = "OldFaithfulApplication")
-public class OldFaithfulApplication implements StreamingApplication
-{
- private final DAG.Locality locality = null;
-
- /**
- * Create the DAG
- */
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
-
- InputGenerator randomInputGenerator = dag.addOperator("rand", new InputGenerator());
- FaithfulRScript rScriptOp = dag.addOperator("rScriptOp", new FaithfulRScript("com/datatorrent/demos/r/oldfaithful/eruptionModel.R", "eruptionModel", "retVal"));
- ConsoleOutputOperator consoles = dag.addOperator("consoles", new ConsoleOutputOperator());
-
- Map<String, FaithfulRScript.REXP_TYPE> argTypeMap = new HashMap<String, FaithfulRScript.REXP_TYPE>();
-
- argTypeMap.put("ELAPSEDTIME", FaithfulRScript.REXP_TYPE.REXP_INT);
- argTypeMap.put("ERUPTIONS", FaithfulRScript.REXP_TYPE.REXP_ARRAY_DOUBLE);
- argTypeMap.put("WAITING", FaithfulRScript.REXP_TYPE.REXP_ARRAY_INT);
-
- rScriptOp.setArgTypeMap(argTypeMap);
-
- dag.addStream("ingen_faithfulRscript", randomInputGenerator.outputPort, rScriptOp.faithfulInput).setLocality(locality);
- dag.addStream("ingen_faithfulRscript_eT", randomInputGenerator.elapsedTime, rScriptOp.inputElapsedTime).setLocality(locality);
- dag.addStream("faithfulRscript_console_s", rScriptOp.strOutput, consoles.input).setLocality(locality);
-
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/r/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/demos/r/src/main/resources/META-INF/properties.xml b/demos/r/src/main/resources/META-INF/properties.xml
deleted file mode 100755
index ec8b070..0000000
--- a/demos/r/src/main/resources/META-INF/properties.xml
+++ /dev/null
@@ -1,41 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<configuration>
-<!--properties for R demo -->
- <property>
- <name>dt.application.OldFaithfulApplication.class</name>
- <value>com.datatorrent.demos.r.oldfaithful.OldFaithfulApplication</value>
- <description>An alias for OldFaithful application</description>
- </property>
-
- <property>
- <name>dt.application.OldFaithfulApplication.operator.*.attr.MEMORY_MB</name>
- <value>1024</value>
- </property>
-
-<!-- Need this to information for loading native libraries -->
- <property>
- <name>dt.attr.CONTAINER_JVM_OPTIONS</name>
- <value>-Djava.library.path=/usr/local/lib/R/site-library/rJava/jri/</value>
- </property>
-
-</configuration>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/r/src/main/resources/com/datatorrent/demos/r/oldfaithful/eruptionModel.R
----------------------------------------------------------------------
diff --git a/demos/r/src/main/resources/com/datatorrent/demos/r/oldfaithful/eruptionModel.R b/demos/r/src/main/resources/com/datatorrent/demos/r/oldfaithful/eruptionModel.R
deleted file mode 100755
index e46fa8d..0000000
--- a/demos/r/src/main/resources/com/datatorrent/demos/r/oldfaithful/eruptionModel.R
+++ /dev/null
@@ -1,60 +0,0 @@
-#!/usr/bin/Rscript
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-
-# This script apply the simple linear regression model for the data set 'faithful',
-# and estimates the next eruption duration given the waiting time since the last eruption.
-#
-
- eruptionModel <- function() {
-
- datavar = data.frame(ERUPTIONS, WAITING)
-
- #attach data variable
- attach(datavar)
-
- #create a linear model using lm(FORMULA, DATAVAR)
- #predict the fall eruption duration (ERUPT) using the waiting time since the last eruption (WAITING)
- eruption.lm <- lm(ERUPTIONS ~ WAITING, datavar)
-
- #display linear model
- eruption.lm
-
- # Get the values of the intercept and unemployment so as to be able to predict the enrolment
- interc<-eruption.lm$coeff[["(Intercept)"]]
- eruptionDuration<-eruption.lm$coeff[["WAITING"]]
-
- # Calculate the enrollment based on the percentage being asked for, and the model that has been rated above.
- nextEruptionDuration<-(interc+(eruptionDuration * ELAPSEDTIME))
-
-retVal<-paste("nextEruptionDuration ", nextEruptionDuration, sep=": ")
-#retVal<-c("interc : ",interc, ", eruptionDuration : ", eruptionDuration,", nextEruptionDuration : ", nextEruptionDuration)
-
-sort( sapply(mget(ls()),object.size) )
-
-detach(datavar);
-
-# Clear all the data from R workspace
-rm(datavar);
-rm(ERUPTIONS);
-rm(WAITING);
-
-return(retVal)
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/r/src/test/java/com/datatorrent/demos/r/oldfaithful/OldFaithfulApplicationTest.java
----------------------------------------------------------------------
diff --git a/demos/r/src/test/java/com/datatorrent/demos/r/oldfaithful/OldFaithfulApplicationTest.java b/demos/r/src/test/java/com/datatorrent/demos/r/oldfaithful/OldFaithfulApplicationTest.java
deleted file mode 100755
index 0bb1901..0000000
--- a/demos/r/src/test/java/com/datatorrent/demos/r/oldfaithful/OldFaithfulApplicationTest.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.r.oldfaithful;
-
-import org.junit.Assert;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.LocalMode;
-
-public class OldFaithfulApplicationTest
-{
-
- private static final Logger LOG = LoggerFactory.getLogger(OldFaithfulApplicationTest.class);
-
- @Test
- public void testSomeMethod() throws Exception
- {
- LocalMode lma = LocalMode.newInstance();
- OldFaithfulApplication app = new OldFaithfulApplication();
- app.populateDAG(lma.getDAG(), new Configuration(false));
-
- try {
- LocalMode.Controller lc = lma.getController();
- lc.setHeartbeatMonitoringEnabled(false);
- lc.run(5000);
- } catch (Exception e) {
- LOG.error("Exception: ", e);
- Assert.fail("Unexpected exception.");
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/r/src/test/resources/dt-site-oldfaithful.xml
----------------------------------------------------------------------
diff --git a/demos/r/src/test/resources/dt-site-oldfaithful.xml b/demos/r/src/test/resources/dt-site-oldfaithful.xml
deleted file mode 100755
index ec8b070..0000000
--- a/demos/r/src/test/resources/dt-site-oldfaithful.xml
+++ /dev/null
@@ -1,41 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<configuration>
-<!--properties for R demo -->
- <property>
- <name>dt.application.OldFaithfulApplication.class</name>
- <value>com.datatorrent.demos.r.oldfaithful.OldFaithfulApplication</value>
- <description>An alias for OldFaithful application</description>
- </property>
-
- <property>
- <name>dt.application.OldFaithfulApplication.operator.*.attr.MEMORY_MB</name>
- <value>1024</value>
- </property>
-
-<!-- Need this to information for loading native libraries -->
- <property>
- <name>dt.attr.CONTAINER_JVM_OPTIONS</name>
- <value>-Djava.library.path=/usr/local/lib/R/site-library/rJava/jri/</value>
- </property>
-
-</configuration>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/r/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/demos/r/src/test/resources/log4j.properties b/demos/r/src/test/resources/log4j.properties
deleted file mode 100755
index cf0d19e..0000000
--- a/demos/r/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,43 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-log4j.rootLogger=DEBUG,CONSOLE
-
-log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
-log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
-log4j.appender.CONSOLE.threshold=${test.log.console.threshold}
-test.log.console.threshold=DEBUG
-
-log4j.appender.RFA=org.apache.log4j.RollingFileAppender
-log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
-log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
-log4j.appender.RFA.File=/tmp/app.log
-
-# to enable, add SYSLOG to rootLogger
-log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender
-log4j.appender.SYSLOG.syslogHost=127.0.0.1
-log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout
-log4j.appender.SYSLOG.layout.conversionPattern=${dt.cid} %-5p [%t] %c{2} %x - %m%n
-log4j.appender.SYSLOG.Facility=LOCAL1
-
-log4j.logger.org=info
-#log4j.logger.org.apache.commons.beanutils=warn
-log4j.logger.com.datatorrent=debug
-log4j.logger.org.apache.apex=debug
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/sql/pom.xml
----------------------------------------------------------------------
diff --git a/demos/sql/pom.xml b/demos/sql/pom.xml
deleted file mode 100644
index 69ffa73..0000000
--- a/demos/sql/pom.xml
+++ /dev/null
@@ -1,102 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>sql-demo</artifactId>
- <packaging>jar</packaging>
-
- <name>Apache Apex Malhar SQL API Demo</name>
- <description>Apex demo applications that use SQL APIs to construct a DAG</description>
-
- <parent>
- <groupId>org.apache.apex</groupId>
- <artifactId>malhar-demos</artifactId>
- <version>3.7.0-SNAPSHOT</version>
- </parent>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <version>1.9.1</version>
- <executions>
- <execution>
- <id>attach-artifacts</id>
- <phase>package</phase>
- <goals>
- <goal>attach-artifact</goal>
- </goals>
- <configuration>
- <artifacts>
- <artifact>
- <file>target/${project.artifactId}-${project.version}.apa</file>
- <type>apa</type>
- </artifact>
- </artifacts>
- <skipAttach>false</skipAttach>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- </plugins>
- </build>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.apex</groupId>
- <artifactId>apex-engine</artifactId>
- <version>${apex.core.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.apex</groupId>
- <artifactId>malhar-sql</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <!-- For KafkaTest -->
- <dependency>
- <groupId>org.apache.apex</groupId>
- <artifactId>malhar-kafka</artifactId>
- <version>${project.parent.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>*</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka_2.11</artifactId>
- <version>0.9.0.0</version>
- <classifier>test</classifier>
- <scope>test</scope>
- </dependency>
-
- </dependencies>
-</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/sql/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/demos/sql/src/assemble/appPackage.xml b/demos/sql/src/assemble/appPackage.xml
deleted file mode 100644
index 4138cf2..0000000
--- a/demos/sql/src/assemble/appPackage.xml
+++ /dev/null
@@ -1,59 +0,0 @@
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
- <id>appPackage</id>
- <formats>
- <format>jar</format>
- </formats>
- <includeBaseDirectory>false</includeBaseDirectory>
- <fileSets>
- <fileSet>
- <directory>${basedir}/target/</directory>
- <outputDirectory>/app</outputDirectory>
- <includes>
- <include>${project.artifactId}-${project.version}.jar</include>
- </includes>
- </fileSet>
- <fileSet>
- <directory>${basedir}/target/deps</directory>
- <outputDirectory>/lib</outputDirectory>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/site/conf</directory>
- <outputDirectory>/conf</outputDirectory>
- <includes>
- <include>*.xml</include>
- </includes>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/main/resources/META-INF</directory>
- <outputDirectory>/META-INF</outputDirectory>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/main/resources/app</directory>
- <outputDirectory>/app</outputDirectory>
- </fileSet>
- </fileSets>
-
-</assembly>
-
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/sql/src/main/java/org/apache/apex/malhar/sql/sample/FusionStyleSQLApplication.java
----------------------------------------------------------------------
diff --git a/demos/sql/src/main/java/org/apache/apex/malhar/sql/sample/FusionStyleSQLApplication.java b/demos/sql/src/main/java/org/apache/apex/malhar/sql/sample/FusionStyleSQLApplication.java
deleted file mode 100644
index 80b997d..0000000
--- a/demos/sql/src/main/java/org/apache/apex/malhar/sql/sample/FusionStyleSQLApplication.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.apex.malhar.sql.sample;
-
-import java.util.Date;
-import java.util.Map;
-
-import org.apache.apex.malhar.kafka.KafkaSinglePortInputOperator;
-import org.apache.apex.malhar.sql.SQLExecEnvironment;
-import org.apache.apex.malhar.sql.table.CSVMessageFormat;
-import org.apache.apex.malhar.sql.table.FileEndpoint;
-import org.apache.apex.malhar.sql.table.StreamEndpoint;
-import org.apache.hadoop.conf.Configuration;
-
-import com.google.common.collect.ImmutableMap;
-
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.DefaultInputPort;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-import com.datatorrent.common.util.BaseOperator;
-import com.datatorrent.contrib.parser.CsvParser;
-
-
-@ApplicationAnnotation(name = "FusionStyleSQLApplication")
-/**
- * @since 3.6.0
- */
-public class FusionStyleSQLApplication implements StreamingApplication
-{
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- SQLExecEnvironment env = SQLExecEnvironment.getEnvironment();
- env.registerFunction("APEXCONCAT", PureStyleSQLApplication.class, "apex_concat_str");
-
- Map<String, Class> fieldMapping = ImmutableMap.<String, Class>of(
- "RowTime", Date.class,
- "id", Integer.class,
- "Product", String.class,
- "units", Integer.class);
-
- // Add Kafka Input
- KafkaSinglePortInputOperator kafkaInput = dag.addOperator("KafkaInput", KafkaSinglePortInputOperator.class);
- kafkaInput.setInitialOffset("EARLIEST");
-
- // Add CSVParser
- CsvParser csvParser = dag.addOperator("CSVParser", CsvParser.class);
- dag.addStream("KafkaToCSV", kafkaInput.outputPort, csvParser.in);
-
- // Register CSV Parser output as input table for first SQL
- env.registerTable(conf.get("sqlSchemaInputName"), new StreamEndpoint(csvParser.out, fieldMapping));
-
- // Register FileEndpoint as output table for second SQL.
- env.registerTable(conf.get("sqlSchemaOutputName"), new FileEndpoint(conf.get("folderPath"),
- conf.get("fileName"), new CSVMessageFormat(conf.get("sqlSchemaOutputDef"))));
-
- // Add second SQL to DAG
- env.executeSQL(dag, conf.get("sql"));
- }
-
- public static class PassThroughOperator extends BaseOperator
- {
- public final transient DefaultOutputPort output = new DefaultOutputPort();
- public final transient DefaultInputPort input = new DefaultInputPort()
- {
- @Override
- public void process(Object o)
- {
- output.emit(output);
- }
- };
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/sql/src/main/java/org/apache/apex/malhar/sql/sample/PureStyleSQLApplication.java
----------------------------------------------------------------------
diff --git a/demos/sql/src/main/java/org/apache/apex/malhar/sql/sample/PureStyleSQLApplication.java b/demos/sql/src/main/java/org/apache/apex/malhar/sql/sample/PureStyleSQLApplication.java
deleted file mode 100644
index 79295f9..0000000
--- a/demos/sql/src/main/java/org/apache/apex/malhar/sql/sample/PureStyleSQLApplication.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.apex.malhar.sql.sample;
-
-import org.apache.apex.malhar.sql.SQLExecEnvironment;
-import org.apache.apex.malhar.sql.table.CSVMessageFormat;
-import org.apache.apex.malhar.sql.table.FileEndpoint;
-import org.apache.apex.malhar.sql.table.KafkaEndpoint;
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-
-@ApplicationAnnotation(name = "PureStyleSQLApplication")
-/**
- * @since 3.6.0
- */
-public class PureStyleSQLApplication implements StreamingApplication
-{
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- // Source definition
- String schemaInName = conf.get("schemaInName");
- String schemaInDef = conf.get("schemaInDef");
- String broker = conf.get("broker");
- String sourceTopic = conf.get("topic");
-
- // Destination definition
- String schemaOutName = conf.get("schemaOutName");
- String schemaOutDef = conf.get("schemaOutDef");
- String outputFolder = conf.get("outputFolder");
- String outFilename = conf.get("destFileName");
-
- // SQL statement
- String sql = conf.get("sql");
-
- SQLExecEnvironment.getEnvironment()
- .registerTable(schemaInName, new KafkaEndpoint(broker, sourceTopic,
- new CSVMessageFormat(schemaInDef)))
- .registerTable(schemaOutName, new FileEndpoint(outputFolder, outFilename,
- new CSVMessageFormat(schemaOutDef)))
- .registerFunction("APEXCONCAT", this.getClass(), "apex_concat_str")
- .executeSQL(dag, sql);
- }
-
- public static String apex_concat_str(String s1, String s2)
- {
- return s1 + s2;
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/sql/src/main/java/org/apache/apex/malhar/sql/sample/SQLApplicationWithAPI.java
----------------------------------------------------------------------
diff --git a/demos/sql/src/main/java/org/apache/apex/malhar/sql/sample/SQLApplicationWithAPI.java b/demos/sql/src/main/java/org/apache/apex/malhar/sql/sample/SQLApplicationWithAPI.java
deleted file mode 100644
index da4f563..0000000
--- a/demos/sql/src/main/java/org/apache/apex/malhar/sql/sample/SQLApplicationWithAPI.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.apex.malhar.sql.sample;
-
-import org.apache.apex.malhar.sql.SQLExecEnvironment;
-import org.apache.apex.malhar.sql.table.CSVMessageFormat;
-import org.apache.apex.malhar.sql.table.FileEndpoint;
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-
-@ApplicationAnnotation(name = "SQLApplicationWithAPI")
-/**
- * @since 3.6.0
- */
-public class SQLApplicationWithAPI implements StreamingApplication
-{
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- // Source definition
- String schemaInName = conf.get("csvSchemaInName");
- String schemaIn = conf.get("csvSchemaIn");
- String sourceFile = conf.get("sourceFile");
-
- SQLExecEnvironment.getEnvironment()
- .registerTable(schemaInName, new FileEndpoint(sourceFile, new CSVMessageFormat(schemaIn)))
- .executeSQL(dag, conf.get("sql"));
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/sql/src/main/java/org/apache/apex/malhar/sql/sample/SQLApplicationWithModelFile.java
----------------------------------------------------------------------
diff --git a/demos/sql/src/main/java/org/apache/apex/malhar/sql/sample/SQLApplicationWithModelFile.java b/demos/sql/src/main/java/org/apache/apex/malhar/sql/sample/SQLApplicationWithModelFile.java
deleted file mode 100644
index 4c90a82..0000000
--- a/demos/sql/src/main/java/org/apache/apex/malhar/sql/sample/SQLApplicationWithModelFile.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.apex.malhar.sql.sample;
-
-import java.io.File;
-import java.io.IOException;
-
-import org.apache.apex.malhar.sql.SQLExecEnvironment;
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-
-@ApplicationAnnotation(name = "SQLApplicationWithModelFile")
-/**
- * @since 3.6.0
- */
-public class SQLApplicationWithModelFile implements StreamingApplication
-{
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- String modelFile = conf.get("modelFile");
- String model;
- try {
- model = FileUtils.readFileToString(new File(modelFile));
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
-
- SQLExecEnvironment.getEnvironment()
- .withModel(model)
- .executeSQL(dag, conf.get("sql"));
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/sql/src/main/resources/META-INF/properties-FusionStyleSQLApplication.xml
----------------------------------------------------------------------
diff --git a/demos/sql/src/main/resources/META-INF/properties-FusionStyleSQLApplication.xml b/demos/sql/src/main/resources/META-INF/properties-FusionStyleSQLApplication.xml
deleted file mode 100644
index 77852e7..0000000
--- a/demos/sql/src/main/resources/META-INF/properties-FusionStyleSQLApplication.xml
+++ /dev/null
@@ -1,65 +0,0 @@
-<?xml version="1.0"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<configuration>
- <!-- Kafka Operator Properties -->
- <property>
- <name>dt.operator.KafkaInput.prop.topics</name>
- <value>dataTopic</value>
- </property>
- <property>
- <name>dt.operator.KafkaInput.prop.clusters</name>
- <value>localhost:9092</value> <!-- broker (NOT zookeeper) address -->
- </property>
-
- <!-- CSV Parser Properties -->
- <property>
- <name>dt.operator.CSVParser.prop.schema</name>
- <value>{"separator":",","quoteChar":"\"","fields":[{"name":"RowTime","type":"Date","constraints":{"format":"dd/MM/yyyy hh:mm:ss Z"}},{"name":"id","type":"Integer"},{"name":"Product","type":"String"},{"name":"units","type":"Integer"}]}</value>
- </property>
-
- <!-- SQL Properties -->
- <property>
- <name>sqlSchemaInputName</name>
- <value>FROMCSV</value>
- </property>
- <property>
- <name>sqlSchemaOutputName</name>
- <value>TOFILE</value>
- </property>
- <property>
- <name>folderPath</name>
- <value>/tmp/output</value>
- </property>
- <property>
- <name>fileName</name>
- <value>output.txt</value>
- </property>
- <property>
- <name>sqlSchemaOutputDef</name>
- <value>{"separator":",","quoteChar":"\"","fields":[{"name":"RowTime1","type":"Date","constraints":{"format":"dd/MM/yyyy hh:mm:ss Z"}},{"name":"RowTime2","type":"Date","constraints":{"format":"dd/MM/yyyy hh:mm:ss Z"}},{"name":"Product","type":"String"}]}</value>
- </property>
- <property>
- <name>sql</name>
- <value>INSERT INTO TOFILE SELECT STREAM ROWTIME, FLOOR(ROWTIME TO DAY), APEXCONCAT('OILPAINT', SUBSTRING(PRODUCT, 6, 7)) FROM FROMCSV WHERE ID > 3 AND PRODUCT LIKE 'paint%'</value>
- </property>
-</configuration>
-
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/sql/src/main/resources/META-INF/properties-PureStyleSQLApplication.xml
----------------------------------------------------------------------
diff --git a/demos/sql/src/main/resources/META-INF/properties-PureStyleSQLApplication.xml b/demos/sql/src/main/resources/META-INF/properties-PureStyleSQLApplication.xml
deleted file mode 100644
index 0d25aa6..0000000
--- a/demos/sql/src/main/resources/META-INF/properties-PureStyleSQLApplication.xml
+++ /dev/null
@@ -1,65 +0,0 @@
-<?xml version="1.0"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<configuration>
- <!-- Input Definition -->
- <property>
- <name>schemaInName</name>
- <value>ORDERS</value>
- </property>
- <property>
- <name>schemaInDef</name>
- <value>{"separator":",","quoteChar":"\"","fields":[{"name":"RowTime","type":"Date","constraints":{"format":"dd/MM/yyyy hh:mm:ss Z"}},{"name":"id","type":"Integer"},{"name":"Product","type":"String"},{"name":"units","type":"Integer"}]}</value>
- </property>
- <property>
- <name>broker</name>
- <value>localhost:9090</value>
- </property>
- <property>
- <name>topic</name>
- <value>inputTopic</value>
- </property>
-
- <!-- Output Definition -->
- <property>
- <name>schemaOutName</name>
- <value>SALES</value>
- </property>
- <property>
- <name>schemaOutDef</name>
- <value>{"separator":",","quoteChar":"\"","fields":[{"name":"RowTime1","type":"Date","constraints":{"format":"dd/MM/yyyy hh:mm:ss Z"}},{"name":"RowTime2","type":"Date","constraints":{"format":"dd/MM/yyyy hh:mm:ss Z"}},{"name":"Product","type":"String"}]}</value>
- </property>
- <property>
- <name>outputFolder</name>
- <value>/tmp/output</value>
- </property>
- <property>
- <name>destFileName</name>
- <value>out.file</value>
- </property>
-
- <!-- Execution SQL -->
- <property>
- <name>sql</name>
- <value>INSERT INTO SALES SELECT STREAM ROWTIME, FLOOR(ROWTIME TO DAY), APEXCONCAT('OILPAINT', SUBSTRING(PRODUCT, 6, 7)) FROM ORDERS WHERE ID > 3 AND PRODUCT LIKE 'paint%'</value>
- </property>
-</configuration>
-
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/sql/src/main/resources/META-INF/properties-SQLApplicationWithAPI.xml
----------------------------------------------------------------------
diff --git a/demos/sql/src/main/resources/META-INF/properties-SQLApplicationWithAPI.xml b/demos/sql/src/main/resources/META-INF/properties-SQLApplicationWithAPI.xml
deleted file mode 100644
index 9ac49d4..0000000
--- a/demos/sql/src/main/resources/META-INF/properties-SQLApplicationWithAPI.xml
+++ /dev/null
@@ -1,43 +0,0 @@
-<?xml version="1.0"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<configuration>
- <!-- Input Definition -->
- <property>
- <name>csvSchemaInName</name>
- <value>ORDERS</value>
- </property>
- <property>
- <name>csvSchemaIn</name>
- <value>{"separator":",","quoteChar":"\"","fields":[{"name":"RowTime","type":"Date","constraints":{"format":"dd/MM/yyyy hh:mm:ss Z"}},{"name":"id","type":"Integer"},{"name":"Product","type":"String"},{"name":"units","type":"Integer"}]}</value>
- </property>
- <property>
- <name>sourceFile</name>
- <value>src/test/resources/input.csv</value>
- </property>
-
- <!-- Execution SQL -->
- <property>
- <name>sql</name>
- <value>SELECT STREAM ROWTIME, PRODUCT FROM ORDERS</value>
- </property>
-</configuration>
-
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/sql/src/main/resources/META-INF/properties-SQLApplicationWithModelFile.xml
----------------------------------------------------------------------
diff --git a/demos/sql/src/main/resources/META-INF/properties-SQLApplicationWithModelFile.xml b/demos/sql/src/main/resources/META-INF/properties-SQLApplicationWithModelFile.xml
deleted file mode 100644
index ab026c2..0000000
--- a/demos/sql/src/main/resources/META-INF/properties-SQLApplicationWithModelFile.xml
+++ /dev/null
@@ -1,32 +0,0 @@
-<?xml version="1.0"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<configuration>
- <property>
- <name>modelFile</name>
- <value>src/main/resources/model/model_file_csv.json</value>
- </property>
- <property>
- <name>sql</name>
- <value>SELECT STREAM ROWTIME, PRODUCT FROM ORDERS</value>
- </property>
-</configuration>
-
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/sql/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/demos/sql/src/main/resources/META-INF/properties.xml b/demos/sql/src/main/resources/META-INF/properties.xml
deleted file mode 100644
index 6080bf6..0000000
--- a/demos/sql/src/main/resources/META-INF/properties.xml
+++ /dev/null
@@ -1,41 +0,0 @@
-<?xml version="1.0"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<configuration>
- <!-- Memory settings for all demos -->
- <property>
- <name>dt.attr.MASTER_MEMORY_MB</name>
- <value>512</value>
- </property>
- <property>
- <name>dt.application.*.operator.*.attr.MEMORY_MB</name>
- <value>256</value>
- </property>
- <property>
- <name>dt.application.*.operator.*.attr.JVM_OPTIONS</name>
- <value>-Xmx128M</value>
- </property>
- <property>
- <name>dt.application.*.operator.*.port.*.attr.BUFFER_MEMORY_MB</name>
- <value>128</value>
- </property>
-</configuration>
-
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/sql/src/main/resources/model/model_file_csv.json
----------------------------------------------------------------------
diff --git a/demos/sql/src/main/resources/model/model_file_csv.json b/demos/sql/src/main/resources/model/model_file_csv.json
deleted file mode 100644
index beba18d..0000000
--- a/demos/sql/src/main/resources/model/model_file_csv.json
+++ /dev/null
@@ -1,27 +0,0 @@
-{
- "version": "1.0",
- "defaultSchema": "APEX",
- "schemas": [{
- "name": "APEX",
- "tables": [
- {
- "name": "ORDERS",
- "type": "custom",
- "factory": "org.apache.apex.malhar.sql.schema.ApexSQLTableFactory",
- "stream": {
- "stream": true
- },
- "operand": {
- "endpoint": "file",
- "messageFormat": "csv",
- "endpointOperands": {
- "directory": "src/test/resources/input.csv"
- },
- "messageFormatOperands": {
- "schema": "{\"separator\":\",\",\"quoteChar\":\"\\\"\",\"fields\":[{\"name\":\"RowTime\",\"type\":\"Date\",\"constraints\":{\"format\":\"dd/MM/yyyy hh:mm:ss\"}},{\"name\":\"id\",\"type\":\"Integer\"},{\"name\":\"Product\",\"type\":\"String\"},{\"name\":\"units\",\"type\":\"Integer\"}]}"
- }
- }
- }
- ]
- }]
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/sql/src/test/java/org/apache/apex/malhar/sql/sample/FusionStyleSQLApplicationTest.java
----------------------------------------------------------------------
diff --git a/demos/sql/src/test/java/org/apache/apex/malhar/sql/sample/FusionStyleSQLApplicationTest.java b/demos/sql/src/test/java/org/apache/apex/malhar/sql/sample/FusionStyleSQLApplicationTest.java
deleted file mode 100644
index 7208701..0000000
--- a/demos/sql/src/test/java/org/apache/apex/malhar/sql/sample/FusionStyleSQLApplicationTest.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.apex.malhar.sql.sample;
-
-import java.io.File;
-import java.util.Arrays;
-import java.util.List;
-import java.util.TimeZone;
-
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TestName;
-
-import org.apache.apex.malhar.kafka.EmbeddedKafka;
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.LocalMode;
-
-public class FusionStyleSQLApplicationTest
-{
- private final String testTopicData = "dataTopic";
- private final String testTopicResult = "resultTopic";
-
- private TimeZone defaultTZ;
- private EmbeddedKafka kafka;
-
- private static String outputFolder = "target/output/";
-
- @Rule
- public TestName testName = new TestName();
-
- @Before
- public void setUp() throws Exception
- {
- defaultTZ = TimeZone.getDefault();
- TimeZone.setDefault(TimeZone.getTimeZone("GMT"));
-
- kafka = new EmbeddedKafka();
- kafka.start();
- kafka.createTopic(testTopicData);
- kafka.createTopic(testTopicResult);
-
- outputFolder += testName.getMethodName() + "/";
- }
-
- @After
- public void tearDown() throws Exception
- {
- kafka.stop();
-
- TimeZone.setDefault(defaultTZ);
- }
-
- @Test
- public void test() throws Exception
- {
- try {
- LocalMode lma = LocalMode.newInstance();
- Configuration conf = new Configuration(false);
- conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
- conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties-FusionStyleSQLApplication.xml"));
-
- conf.set("dt.operator.KafkaInput.prop.topics", testTopicData);
- conf.set("dt.operator.KafkaInput.prop.clusters", kafka.getBroker());
- conf.set("folderPath", outputFolder);
- conf.set("fileName", "out.tmp");
-
- FusionStyleSQLApplication app = new FusionStyleSQLApplication();
-
- lma.prepareDAG(app, conf);
-
- LocalMode.Controller lc = lma.getController();
-
- lc.runAsync();
- kafka.publish(testTopicData, Arrays.asList("15/02/2016 10:15:00 +0000,1,paint1,11",
- "15/02/2016 10:16:00 +0000,2,paint2,12",
- "15/02/2016 10:17:00 +0000,3,paint3,13", "15/02/2016 10:18:00 +0000,4,paint4,14",
- "15/02/2016 10:19:00 +0000,5,paint5,15", "15/02/2016 10:10:00 +0000,6,abcde6,16"));
-
- Assert.assertTrue(PureStyleSQLApplicationTest.waitTillFileIsPopulated(outputFolder, 40000));
- lc.shutdown();
-
- File file = new File(outputFolder);
- File file1 = new File(outputFolder + file.list()[0]);
- List<String> strings = FileUtils.readLines(file1);
-
- String[] actualLines = strings.toArray(new String[strings.size()]);
- String[] expectedLines = new String[] {
- "15/02/2016 10:18:00 +0000,15/02/2016 12:00:00 +0000,OILPAINT4",
- "",
- "15/02/2016 10:19:00 +0000,15/02/2016 12:00:00 +0000,OILPAINT5",
- ""};
- Assert.assertEquals(expectedLines.length, actualLines.length);
- for (int i = 0; i < actualLines.length; i++) {
- Assert.assertEquals(expectedLines[i], actualLines[i]);
- }
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/sql/src/test/java/org/apache/apex/malhar/sql/sample/PureStyleSQLApplicationTest.java
----------------------------------------------------------------------
diff --git a/demos/sql/src/test/java/org/apache/apex/malhar/sql/sample/PureStyleSQLApplicationTest.java b/demos/sql/src/test/java/org/apache/apex/malhar/sql/sample/PureStyleSQLApplicationTest.java
deleted file mode 100644
index f298059..0000000
--- a/demos/sql/src/test/java/org/apache/apex/malhar/sql/sample/PureStyleSQLApplicationTest.java
+++ /dev/null
@@ -1,155 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.apex.malhar.sql.sample;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
-import java.util.TimeZone;
-
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TestName;
-
-import org.apache.apex.malhar.kafka.EmbeddedKafka;
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-
-import com.google.common.collect.Lists;
-
-import com.datatorrent.api.LocalMode;
-
-
-public class PureStyleSQLApplicationTest
-{
- private final String testTopicData = "dataTopic";
- private final String testTopicResult = "resultTopic";
-
- private TimeZone defaultTZ;
- private EmbeddedKafka kafka;
- private static String outputFolder = "target/output/";
-
- @Rule
- public TestName testName = new TestName();
-
- @Before
- public void setUp() throws Exception
- {
- defaultTZ = TimeZone.getDefault();
- TimeZone.setDefault(TimeZone.getTimeZone("GMT"));
-
- kafka = new EmbeddedKafka();
- kafka.start();
- kafka.createTopic(testTopicData);
- kafka.createTopic(testTopicResult);
-
- outputFolder += testName.getMethodName() + "/";
- }
-
- @After
- public void tearDown() throws Exception
- {
- kafka.stop();
- TimeZone.setDefault(defaultTZ);
- }
-
- @Test
- public void test() throws Exception
- {
- LocalMode lma = LocalMode.newInstance();
- Configuration conf = new Configuration(false);
- conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
- conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties-PureStyleSQLApplication.xml"));
-
- conf.set("broker", kafka.getBroker());
- conf.set("topic", testTopicData);
- conf.set("outputFolder", outputFolder);
- conf.set("destFileName", "out.tmp");
-
- PureStyleSQLApplication app = new PureStyleSQLApplication();
-
- lma.prepareDAG(app, conf);
-
- LocalMode.Controller lc = lma.getController();
-
- lc.runAsync();
- kafka.publish(testTopicData, Arrays.asList(
- "15/02/2016 10:15:00 +0000,1,paint1,11",
- "15/02/2016 10:16:00 +0000,2,paint2,12",
- "15/02/2016 10:17:00 +0000,3,paint3,13",
- "15/02/2016 10:18:00 +0000,4,paint4,14",
- "15/02/2016 10:19:00 +0000,5,paint5,15",
- "15/02/2016 10:10:00 +0000,6,abcde6,16"));
-
- Assert.assertTrue(waitTillFileIsPopulated(outputFolder, 40000));
- lc.shutdown();
-
- File file = new File(outputFolder);
- File file1 = new File(outputFolder + file.list()[0]);
- List<String> strings = FileUtils.readLines(file1);
-
- String[] actualLines = strings.toArray(new String[strings.size()]);
-
- String[] expectedLines = new String[]{
- "15/02/2016 10:18:00 +0000,15/02/2016 12:00:00 +0000,OILPAINT4",
- "",
- "15/02/2016 10:19:00 +0000,15/02/2016 12:00:00 +0000,OILPAINT5",
- ""};
-
- Assert.assertEquals(expectedLines.length, actualLines.length);
- for (int i = 0;i < expectedLines.length; i++) {
- Assert.assertEquals(expectedLines[i], actualLines[i]);
- }
- }
-
- public static boolean waitTillFileIsPopulated(String outputFolder, int timeout) throws IOException, InterruptedException
- {
- boolean result;
- long now = System.currentTimeMillis();
- Path outDir = new Path("file://" + new File(outputFolder).getAbsolutePath());
- try (FileSystem fs = FileSystem.newInstance(outDir.toUri(), new Configuration())) {
- List<String> strings = Lists.newArrayList();
- while (System.currentTimeMillis() - now < timeout) {
- if (fs.exists(outDir)) {
- File file = new File(outputFolder);
- if (file.list().length > 0) {
- File file1 = new File(outputFolder + file.list()[0]);
- strings = FileUtils.readLines(file1);
- if (strings.size() != 0) {
- break;
- }
- }
- }
-
- Thread.sleep(500);
- }
-
- result = fs.exists(outDir) && (strings.size() != 0);
- }
-
- return result;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/sql/src/test/java/org/apache/apex/malhar/sql/sample/SQLApplicationWithAPITest.java
----------------------------------------------------------------------
diff --git a/demos/sql/src/test/java/org/apache/apex/malhar/sql/sample/SQLApplicationWithAPITest.java b/demos/sql/src/test/java/org/apache/apex/malhar/sql/sample/SQLApplicationWithAPITest.java
deleted file mode 100644
index 6b1a404..0000000
--- a/demos/sql/src/test/java/org/apache/apex/malhar/sql/sample/SQLApplicationWithAPITest.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.apex.malhar.sql.sample;
-
-import java.io.ByteArrayOutputStream;
-import java.io.PrintStream;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.TimeZone;
-
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import org.apache.hadoop.conf.Configuration;
-
-import com.google.common.base.Predicates;
-import com.google.common.collect.Collections2;
-
-import com.datatorrent.api.LocalMode;
-
-
-public class SQLApplicationWithAPITest
-{
- private TimeZone defaultTZ;
-
- @Before
- public void setUp() throws Exception
- {
- defaultTZ = TimeZone.getDefault();
- TimeZone.setDefault(TimeZone.getTimeZone("GMT"));
- }
-
- @After
- public void tearDown() throws Exception
- {
- TimeZone.setDefault(defaultTZ);
- }
-
- @Test
- public void test() throws Exception
- {
- LocalMode lma = LocalMode.newInstance();
- Configuration conf = new Configuration(false);
- conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
- conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties-SQLApplicationWithAPI.xml"));
-
- SQLApplicationWithAPI app = new SQLApplicationWithAPI();
-
- lma.prepareDAG(app, conf);
-
- LocalMode.Controller lc = lma.getController();
-
- PrintStream originalSysout = System.out;
- final ByteArrayOutputStream baos = new ByteArrayOutputStream();
- System.setOut(new PrintStream(baos));
-
- lc.runAsync();
- SQLApplicationWithModelFileTest.waitTillStdoutIsPopulated(baos, 30000);
- lc.shutdown();
-
- System.setOut(originalSysout);
-
- String[] sout = baos.toString().split(System.lineSeparator());
- Collection<String> filter = Collections2.filter(Arrays.asList(sout), Predicates.containsPattern("Delta Record:"));
-
- String[] actualLines = filter.toArray(new String[filter.size()]);
- Assert.assertTrue(actualLines[0].contains("RowTime=Mon Feb 15 10:15:00 GMT 2016, Product=paint1"));
- Assert.assertTrue(actualLines[1].contains("RowTime=Mon Feb 15 10:16:00 GMT 2016, Product=paint2"));
- Assert.assertTrue(actualLines[2].contains("RowTime=Mon Feb 15 10:17:00 GMT 2016, Product=paint3"));
- Assert.assertTrue(actualLines[3].contains("RowTime=Mon Feb 15 10:18:00 GMT 2016, Product=paint4"));
- Assert.assertTrue(actualLines[4].contains("RowTime=Mon Feb 15 10:19:00 GMT 2016, Product=paint5"));
- Assert.assertTrue(actualLines[5].contains("RowTime=Mon Feb 15 10:10:00 GMT 2016, Product=abcde6"));
- }
-}
[03/30] apex-malhar git commit: Renamed demos to examples. Packages
and artifactid names are changed as suggested.
Posted by th...@apache.org.
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/sql/src/test/java/org/apache/apex/malhar/sql/sample/SQLApplicationWithModelFileTest.java
----------------------------------------------------------------------
diff --git a/examples/sql/src/test/java/org/apache/apex/malhar/sql/sample/SQLApplicationWithModelFileTest.java b/examples/sql/src/test/java/org/apache/apex/malhar/sql/sample/SQLApplicationWithModelFileTest.java
new file mode 100644
index 0000000..7bbb8ec
--- /dev/null
+++ b/examples/sql/src/test/java/org/apache/apex/malhar/sql/sample/SQLApplicationWithModelFileTest.java
@@ -0,0 +1,113 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.malhar.sql.sample;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.TimeZone;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.google.common.base.Predicates;
+import com.google.common.collect.Collections2;
+import com.google.common.collect.Lists;
+
+import com.datatorrent.api.LocalMode;
+
+public class SQLApplicationWithModelFileTest
+{
+ private TimeZone defaultTZ;
+
+ @Before
+ public void setUp() throws Exception
+ {
+ defaultTZ = TimeZone.getDefault();
+ TimeZone.setDefault(TimeZone.getTimeZone("GMT"));
+ }
+
+ @After
+ public void tearDown() throws Exception
+ {
+ TimeZone.setDefault(defaultTZ);
+ }
+
+ @Test
+ public void test() throws Exception
+ {
+ LocalMode lma = LocalMode.newInstance();
+ Configuration conf = new Configuration(false);
+ conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
+ conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties-SQLApplicationWithModelFile.xml"));
+
+ SQLApplicationWithModelFile app = new SQLApplicationWithModelFile();
+
+ lma.prepareDAG(app, conf);
+
+ LocalMode.Controller lc = lma.getController();
+
+ PrintStream originalSysout = System.out;
+ final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ System.setOut(new PrintStream(baos));
+
+ lc.runAsync();
+ waitTillStdoutIsPopulated(baos, 30000);
+ lc.shutdown();
+
+ System.setOut(originalSysout);
+
+ String[] sout = baos.toString().split(System.lineSeparator());
+ Collection<String> filter = Collections2.filter(Arrays.asList(sout), Predicates.containsPattern("Delta Record:"));
+
+ String[] actualLines = filter.toArray(new String[filter.size()]);
+ Assert.assertTrue(actualLines[0].contains("RowTime=Mon Feb 15 10:15:00 GMT 2016, Product=paint1"));
+ Assert.assertTrue(actualLines[1].contains("RowTime=Mon Feb 15 10:16:00 GMT 2016, Product=paint2"));
+ Assert.assertTrue(actualLines[2].contains("RowTime=Mon Feb 15 10:17:00 GMT 2016, Product=paint3"));
+ Assert.assertTrue(actualLines[3].contains("RowTime=Mon Feb 15 10:18:00 GMT 2016, Product=paint4"));
+ Assert.assertTrue(actualLines[4].contains("RowTime=Mon Feb 15 10:19:00 GMT 2016, Product=paint5"));
+ Assert.assertTrue(actualLines[5].contains("RowTime=Mon Feb 15 10:10:00 GMT 2016, Product=abcde6"));
+ }
+
+ public static boolean waitTillStdoutIsPopulated(ByteArrayOutputStream baos, int timeout) throws InterruptedException,
+ IOException
+ {
+ long now = System.currentTimeMillis();
+ Collection<String> filter = Lists.newArrayList();
+ while (System.currentTimeMillis() - now < timeout) {
+ baos.flush();
+ String[] sout = baos.toString().split(System.lineSeparator());
+ filter = Collections2.filter(Arrays.asList(sout), Predicates.containsPattern("Delta Record:"));
+ if (filter.size() != 0) {
+ break;
+ }
+
+ Thread.sleep(500);
+ }
+
+ return (filter.size() != 0);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/sql/src/test/resources/input.csv
----------------------------------------------------------------------
diff --git a/examples/sql/src/test/resources/input.csv b/examples/sql/src/test/resources/input.csv
new file mode 100644
index 0000000..c4786d1
--- /dev/null
+++ b/examples/sql/src/test/resources/input.csv
@@ -0,0 +1,6 @@
+15/02/2016 10:15:00 +0000,1,paint1,11
+15/02/2016 10:16:00 +0000,2,paint2,12
+15/02/2016 10:17:00 +0000,3,paint3,13
+15/02/2016 10:18:00 +0000,4,paint4,14
+15/02/2016 10:19:00 +0000,5,paint5,15
+15/02/2016 10:10:00 +0000,6,abcde6,16
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/sql/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/examples/sql/src/test/resources/log4j.properties b/examples/sql/src/test/resources/log4j.properties
new file mode 100644
index 0000000..8ea3cfe
--- /dev/null
+++ b/examples/sql/src/test/resources/log4j.properties
@@ -0,0 +1,50 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+log4j.rootLogger=DEBUG,CONSOLE
+
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+log4j.appender.CONSOLE.threshold=WARN
+test.log.console.threshold=WARN
+
+log4j.appender.RFA=org.apache.log4j.RollingFileAppender
+log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
+log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+log4j.appender.RFA.File=/tmp/app.log
+
+# to enable, add SYSLOG to rootLogger
+log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender
+log4j.appender.SYSLOG.syslogHost=127.0.0.1
+log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout
+log4j.appender.SYSLOG.layout.conversionPattern=${dt.cid} %-5p [%t] %c{2} %x - %m%n
+log4j.appender.SYSLOG.Facility=LOCAL1
+
+log4j.logger.org=info
+#log4j.logger.org.apache.commons.beanutils=warn
+log4j.logger.com.datatorrent=INFO
+log4j.logger.org.apache.apex=INFO
+
+log4j.logger.org.apache.calcite=WARN
+log4j.logger.org.apache.kafka=WARN
+log4j.logger.org.I0Itec.zkclient.ZkClient=WARN
+log4j.logger.org.apache.zookeeper=WARN
+log4j.logger.kafka=WARN
+log4j.logger.kafka.consumer=WARN
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/twitter/pom.xml
----------------------------------------------------------------------
diff --git a/examples/twitter/pom.xml b/examples/twitter/pom.xml
new file mode 100644
index 0000000..76924c9
--- /dev/null
+++ b/examples/twitter/pom.xml
@@ -0,0 +1,101 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>malhar-examples-twitter</artifactId>
+ <packaging>jar</packaging>
+
+ <name>Apache Apex Malhar Twitter Example</name>
+ <description>Twitter Rolling Top Words application demonstrates real-time computations over a sliding window.</description>
+
+ <parent>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-examples</artifactId>
+ <version>3.7.0-SNAPSHOT</version>
+ </parent>
+
+ <properties>
+ <skipTests>true</skipTests>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <!-- required by twitter example -->
+ <groupId>org.twitter4j</groupId>
+ <artifactId>twitter4j-core</artifactId>
+ <version>4.0.6</version>
+ </dependency>
+ <dependency>
+ <!-- required by twitter example -->
+ <groupId>org.twitter4j</groupId>
+ <artifactId>twitter4j-stream</artifactId>
+ <version>4.0.6</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase</artifactId>
+ <version>0.94.20</version>
+ <type>jar</type>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>mysql</groupId>
+ <artifactId>mysql-connector-java</artifactId>
+ <version>5.1.22</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-contrib</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk-kinesis</artifactId>
+ <version>1.9.10</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ <version>2.4.4</version>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>it.unimi.dsi</groupId>
+ <artifactId>fastutil</artifactId>
+ <version>6.6.4</version>
+ </dependency>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/twitter/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/examples/twitter/src/assemble/appPackage.xml b/examples/twitter/src/assemble/appPackage.xml
new file mode 100644
index 0000000..4138cf2
--- /dev/null
+++ b/examples/twitter/src/assemble/appPackage.xml
@@ -0,0 +1,59 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+ <id>appPackage</id>
+ <formats>
+ <format>jar</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>${basedir}/target/</directory>
+ <outputDirectory>/app</outputDirectory>
+ <includes>
+ <include>${project.artifactId}-${project.version}.jar</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/target/deps</directory>
+ <outputDirectory>/lib</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/site/conf</directory>
+ <outputDirectory>/conf</outputDirectory>
+ <includes>
+ <include>*.xml</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/META-INF</directory>
+ <outputDirectory>/META-INF</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/app</directory>
+ <outputDirectory>/app</outputDirectory>
+ </fileSet>
+ </fileSets>
+
+</assembly>
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/twitter/src/main/java/org/apache/apex/examples/twitter/KinesisHashtagsApplication.java
----------------------------------------------------------------------
diff --git a/examples/twitter/src/main/java/org/apache/apex/examples/twitter/KinesisHashtagsApplication.java b/examples/twitter/src/main/java/org/apache/apex/examples/twitter/KinesisHashtagsApplication.java
new file mode 100644
index 0000000..be7edfb
--- /dev/null
+++ b/examples/twitter/src/main/java/org/apache/apex/examples/twitter/KinesisHashtagsApplication.java
@@ -0,0 +1,236 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.twitter;
+
+import java.net.URI;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.DAG.Locality;
+import com.datatorrent.api.Operator.InputPort;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.contrib.kinesis.AbstractKinesisInputOperator;
+import com.datatorrent.contrib.kinesis.KinesisStringInputOperator;
+import com.datatorrent.contrib.kinesis.KinesisStringOutputOperator;
+import com.datatorrent.contrib.kinesis.ShardManager;
+import com.datatorrent.contrib.twitter.TwitterSampleInput;
+import com.datatorrent.lib.algo.UniqueCounter;
+import com.datatorrent.lib.io.ConsoleOutputOperator;
+import com.datatorrent.lib.io.PubSubWebSocketOutputOperator;
+
+/**
+ * Twitter Example Application: <br>
+ * This example application samples random public status from twitter, send to Hashtag
+ * extractor and extract the status and send it into kinesis <br>
+ * Get the records from kinesis and converts into Hashtags. Top 10 Hashtag(s) mentioned in
+ * tweets in last 5 mins are displayed on every window count (500ms).<br>
+ * <br>
+ *
+ * Real Time Calculation :<br>
+ * This application calculates top 10 Hashtag mentioned in tweets in last 5
+ * minutes across a 1% random tweet sampling on a rolling window basis.<br>
+ * <br>
+ * Before running this application, you need to have a <a href="https://dev.twitter.com/apps">Twitter API account</a>,
+ * <a href="https://http://aws.amazon.com/">AWS Account</a> and configure the authentication details.
+ * For launch from CLI, those go into ~/.dt/dt-site.xml:
+ * <pre>
+ * {@code
+ * <?xml version="1.0" encoding="UTF-8"?>
+ * <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+ * <configuration>
+ *
+ * <property> <name>dt.operator.TweetSampler.prop.consumerKey</name>
+ * <value>TBD</value> </property>
+ *
+ * <property> <name>dt.operator.TweetSampler.prop.consumerSecret</name>
+ * <value>TBD</value> </property>
+ *
+ * <property> <name>dt.operator.TweetSampler.prop.accessToken</name>
+ * <value>TBD</value> </property>
+ *
+ * <property> <name>dt.operator.TweetSampler.prop.accessTokenSecret</name>
+ * <value>TBD</value> </property>
+ *
+ * <property> <name>dt.operator.FromKinesis.streamName</name>
+ * <value>TBD</value> </property>
+ *
+ * <property> <name>dt.operator.FromKinesis.accessKey</name>
+ * <value>TBD</value> </property>
+ *
+ * <property> <name>dt.operator.FromKinesis.secretKey</name>
+ * <value>TBD</value> </property>
+ *
+ * <property> <name>dt.operator.ToKinesis.streamName</name>
+ * <value>TBD</value> </property>
+ *
+ * <property> <name>dt.operator.ToKinesis.accessKey</name>
+ * <value>TBD</value> </property>
+ *
+ * <property> <name>dt.operator.ToKinesis.secretKey</name>
+ * <value>TBD</value> </property>
+ *
+ * </configuration>
+ * }
+ * </pre>
+ * Custom Attributes: <br>
+ * <b>topCounts operator : <b>
+ * <ul>
+ * <li>Top Count : 10, number of top unique Hashtag to be reported.</li>
+ * <li>Sliding window count : 600, report over last 5 min (600 * .5 / 60 mins)</li>
+ * <li>window slide value : 1</li>
+ * </ul>
+ * <p>
+ * Running Java Test or Main app in IDE:
+ *
+ * <pre>
+ * LocalMode.runApp(new Application(), 600000); // 10 min run
+ * </pre>
+ *
+ * Run Success : <br>
+ * For successful deployment and run, user should see similar output on console as below:
+ *
+ * <pre>
+ * topHashtags: {"string": "{\"gameinsight\":\"12\",\"android\":\"8\",\"WotaJKT48alay\":\"12\",\"TernyataJKT48CyberItuForoumSampah\":\"8\",\"NHL15Subban\":\"30\",\"JKT48CYBERKeracunanCUKRIK\":\"8\",\"verifydjzoodel\":\"59\",\"androidgames\":\"9\",\"\u0645\u0639_\u0627\u0644\u0644\u0647\":\"10\",\"Jct48cyberTAIBABI\":\"10\"}"}
+ * topHashtags: {"string": "{\"gameinsight\":\"12\",\"android\":\"8\",\"WotaJKT48alay\":\"12\",\"TernyataJKT48CyberItuForoumSampah\":\"8\",\"NHL15Subban\":\"30\",\"JKT48CYBERKeracunanCUKRIK\":\"8\",\"verifydjzoodel\":\"59\",\"androidgames\":\"9\",\"\u0645\u0639_\u0627\u0644\u0644\u0647\":\"10\",\"Jct48cyberTAIBABI\":\"10\"}"}
+ * topHashtags: {"string": "{\"gameinsight\":\"12\",\"android\":\"8\",\"WotaJKT48alay\":\"12\",\"TernyataJKT48CyberItuForoumSampah\":\"8\",\"NHL15Subban\":\"30\",\"JKT48CYBERKeracunanCUKRIK\":\"8\",\"verifydjzoodel\":\"59\",\"androidgames\":\"9\",\"\u0645\u0639_\u0627\u0644\u0644\u0647\":\"10\",\"Jct48cyberTAIBABI\":\"10\"}"}
+ * topHashtags: {"string": "{\"gameinsight\":\"12\",\"android\":\"8\",\"WotaJKT48alay\":\"12\",\"TernyataJKT48CyberItuForoumSampah\":\"8\",\"NHL15Subban\":\"30\",\"JKT48CYBERKeracunanCUKRIK\":\"8\",\"verifydjzoodel\":\"59\",\"androidgames\":\"9\",\"\u0645\u0639_\u0627\u0644\u0644\u0647\":\"10\",\"Jct48cyberTAIBABI\":\"10\"}"}
+ * topHashtags: {"string": "{\"gameinsight\":\"12\",\"android\":\"8\",\"WotaJKT48alay\":\"12\",\"TernyataJKT48CyberItuForoumSampah\":\"8\",\"NHL15Subban\":\"30\",\"JKT48CYBERKeracunanCUKRIK\":\"8\",\"verifydjzoodel\":\"59\",\"androidgames\":\"9\",\"\u0645\u0639_\u0627\u0644\u0644\u0647\":\"10\",\"Jct48cyberTAIBABI\":\"10\"}"}
+ * topHashtags: {"string": "{\"gameinsight\":\"12\",\"android\":\"8\",\"WotaJKT48alay\":\"12\",\"TernyataJKT48CyberItuForoumSampah\":\"8\",\"NHL15Subban\":\"30\",\"JKT48CYBERKeracunanCUKRIK\":\"8\",\"verifydjzoodel\":\"59\",\"androidgames\":\"9\",\"\u0645\u0639_\u0627\u0644\u0644\u0647\":\"10\",\"Jct48cyberTAIBABI\":\"10\"}"}
+ * topHashtags: {"string": "{\"gameinsight\":\"12\",\"android\":\"8\",\"WotaJKT48alay\":\"12\",\"TernyataJKT48CyberItuForoumSampah\":\"8\",\"NHL15Subban\":\"30\",\"JKT48CYBERKeracunanCUKRIK\":\"8\",\"verifydjzoodel\":\"59\",\"androidgames\":\"9\",\"\u0645\u0639_\u0627\u0644\u0644\u0647\":\"10\",\"Jct48cyberTAIBABI\":\"10\"}"}
+ * topHashtags: {"string": "{\"gameinsight\":\"12\",\"android\":\"8\",\"WotaJKT48alay\":\"12\",\"TernyataJKT48CyberItuForoumSampah\":\"8\",\"NHL15Subban\":\"30\",\"JKT48CYBERKeracunanCUKRIK\":\"8\",\"verifydjzoodel\":\"59\",\"androidgames\":\"9\",\"\u0645\u0639_\u0627\u0644\u0644\u0647\":\"10\",\"Jct48cyberTAIBABI\":\"10\"}"}
+ * topHashtags: {"string": "{\"gameinsight\":\"12\",\"android\":\"8\",\"WotaJKT48alay\":\"12\",\"TernyataJKT48CyberItuForoumSampah\":\"8\",\"NHL15Subban\":\"30\",\"JKT48CYBERKeracunanCUKRIK\":\"8\",\"verifydjzoodel\":\"59\",\"androidgames\":\"9\",\"\u0645\u0639_\u0627\u0644\u0644\u0647\":\"10\",\"Jct48cyberTAIBABI\":\"10\"}"}
+ * topHashtags: {"string": "{\"gameinsight\":\"12\",\"android\":\"8\",\"WotaJKT48alay\":\"12\",\"TernyataJKT48CyberItuForoumSampah\":\"8\",\"NHL15Subban\":\"30\",\"JKT48CYBERKeracunanCUKRIK\":\"8\",\"verifydjzoodel\":\"59\",\"androidgames\":\"9\",\"\u0645\u0639_\u0627\u0644\u0644\u0647\":\"10\",\"Jct48cyberTAIBABI\":\"10\"}"}
+ * 2013-06-17 14:38:55,201 [main] INFO stram.StramLocalCluster run - Application finished.
+ * 2013-06-17 14:38:55,201 [container-2] INFO stram.StramChild processHeartbeatResponse - Received shutdown request
+ * </pre>
+ *
+ * Scaling Options : <br>
+ * User can scale application by setting intial partition size > 1 on count
+ * unique operator. <br>
+ * <br>
+ *
+ * Application DAG : <br>
+ * <img src="doc-files/Application.gif" width=600px > <br>
+ * <br>
+ *
+ * Streaming Window Size : 500ms(default) <br>
+ * Operator Details : <br>
+ * <ul>
+ * <li><b>The twitterFeed operator : </b> This operator samples random public
+ * statues from twitter and emits to application. <br>
+ * Class : org.apache.apex.examples.twitter.TwitterSampleInput <br>
+ * StateFull : No, window count 1 <br>
+ * </li>
+ * <li><b>The HashtagExtractor operator : </b> This operator extracts Hashtag from
+ * random sampled statues from twitter. <br>
+ * Class : {@link TwitterStatusHashtagExtractor} <br>
+ * StateFull : No, window count 1 <br>
+ * </li>
+ * <li><b>The outputOp operator : </b> This operator sent the tags into the kinesis. <br>
+ * Class : {@link com.datatorrent.contrib.kinesis.KinesisStringOutputOperator} <br>
+ * </li>
+ * <li><b>The inputOp operator : </b> This operator fetches the records from kinesis and
+ * converts into hastags and emits them. <br>
+ * Class : {@link com.datatorrent.contrib.kinesis.KinesisStringOutputOperator} <br>
+ * </li>
+ * <li><b>The uniqueCounter operator : </b> This operator aggregates count for each
+ * Hashtag extracted from random samples. <br>
+ * Class : {@link com.datatorrent.lib.algo.UniqueCounter} <br>
+ * StateFull : No, window count 1 <br>
+ * </li>
+ * <li><b> The topCounts operator : </b> This operator caluculates top Hashtag in last 1
+ * min sliding window count 1. <br>
+ * Class : com.datatorrent.lib.algo.WindowedTopCounter <br>
+ * StateFull : Yes, sliding window count 120 (1 min) <br>
+ * </li>
+ * <li><b>The operator Console: </b> This operator just outputs the input tuples
+ * to the console (or stdout). <br>
+ * </li>
+ * </ul>
+ *
+ * @since 2.0.0
+ */
+@ApplicationAnnotation(name = "TwitterKinesisExample")
+public class KinesisHashtagsApplication implements StreamingApplication
+{
+ private final Locality locality = null;
+
+ private InputPort<Object> consoleOutput(DAG dag, String operatorName)
+ {
+ String gatewayAddress = dag.getValue(DAG.GATEWAY_CONNECT_ADDRESS);
+ if (!StringUtils.isEmpty(gatewayAddress)) {
+ URI uri = URI.create("ws://" + gatewayAddress + "/pubsub");
+ String topic = "examples.twitter." + operatorName;
+ //LOG.info("WebSocket with gateway at: {}", gatewayAddress);
+ PubSubWebSocketOutputOperator<Object> wsOut = dag.addOperator(operatorName, new PubSubWebSocketOutputOperator<Object>());
+ wsOut.setUri(uri);
+ wsOut.setTopic(topic);
+ return wsOut.input;
+ }
+ ConsoleOutputOperator operator = dag.addOperator(operatorName, new ConsoleOutputOperator());
+ operator.setStringFormat(operatorName + ": %s");
+ return operator.input;
+ }
+
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ // Setup the operator to get the data from twitter sample stream injected into the system.
+ TwitterSampleInput twitterFeed = new TwitterSampleInput();
+ twitterFeed = dag.addOperator("TweetSampler", twitterFeed);
+
+ // Setup the operator to get the Hashtags extracted from the twitter statuses
+ TwitterStatusHashtagExtractor HashtagExtractor = dag.addOperator("HashtagExtractor", TwitterStatusHashtagExtractor.class);
+
+ //Setup the operator send the twitter statuses to kinesis
+ KinesisStringOutputOperator outputOp = dag.addOperator("ToKinesis", new KinesisStringOutputOperator());
+ outputOp.setBatchSize(500);
+
+ // Feed the statuses from feed into the input of the Hashtag extractor.
+ dag.addStream("TweetStream", twitterFeed.status, HashtagExtractor.input).setLocality(Locality.CONTAINER_LOCAL);
+ // Start counting the Hashtags coming out of Hashtag extractor
+ dag.addStream("SendToKinesis", HashtagExtractor.hashtags, outputOp.inputPort).setLocality(locality);
+
+ //------------------------------------------------------------------------------------------
+
+ KinesisStringInputOperator inputOp = dag.addOperator("FromKinesis", new KinesisStringInputOperator());
+ ShardManager shardStats = new ShardManager();
+ inputOp.setShardManager(shardStats);
+ inputOp.getConsumer().setRecordsLimit(600);
+ inputOp.setStrategy(AbstractKinesisInputOperator.PartitionStrategy.MANY_TO_ONE.toString());
+
+ // Setup a node to count the unique Hashtags within a window.
+ UniqueCounter<String> uniqueCounter = dag.addOperator("UniqueHashtagCounter", new UniqueCounter<String>());
+
+ // Get the aggregated Hashtag counts and count them over last 5 mins.
+ WindowedTopCounter<String> topCounts = dag.addOperator("TopCounter", new WindowedTopCounter<String>());
+ topCounts.setTopCount(10);
+ topCounts.setSlidingWindowWidth(600);
+ topCounts.setDagWindowWidth(1);
+
+ dag.addStream("TwittedHashtags", inputOp.outputPort, uniqueCounter.data).setLocality(locality);
+
+ // Count unique Hashtags
+ dag.addStream("UniqueHashtagCounts", uniqueCounter.count, topCounts.input).setLocality(locality);
+ // Count top 10
+ dag.addStream("TopHashtags", topCounts.output, consoleOutput(dag, "topHashtags")).setLocality(locality);
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/twitter/src/main/java/org/apache/apex/examples/twitter/SlidingContainer.java
----------------------------------------------------------------------
diff --git a/examples/twitter/src/main/java/org/apache/apex/examples/twitter/SlidingContainer.java b/examples/twitter/src/main/java/org/apache/apex/examples/twitter/SlidingContainer.java
new file mode 100644
index 0000000..dfcac81
--- /dev/null
+++ b/examples/twitter/src/main/java/org/apache/apex/examples/twitter/SlidingContainer.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.twitter;
+
+import java.io.Serializable;
+
+/**
+ * Developed for a example<br>
+ *
+ * @param <T> Type of object for which sliding window is being maintained.
+ * @since 0.3.2
+ */
+public class SlidingContainer<T> implements Serializable
+{
+ private static final long serialVersionUID = 201305291751L;
+ T identifier;
+ int totalCount;
+ int position;
+ int[] windowedCount;
+
+ @SuppressWarnings("unused")
+ private SlidingContainer()
+ {
+ /* needed for Kryo serialization */
+ }
+
+ public SlidingContainer(T identifier, int windowCount)
+ {
+ this.identifier = identifier;
+ this.totalCount = 0;
+ this.position = 0;
+ windowedCount = new int[windowCount];
+ }
+
+ public void adjustCount(int i)
+ {
+ windowedCount[position] += i;
+ }
+
+ public void slide()
+ {
+ int currentCount = windowedCount[position];
+ position = position == windowedCount.length - 1 ? 0 : position + 1;
+ totalCount += currentCount - windowedCount[position];
+ windowedCount[position] = 0;
+ }
+
+ @Override
+ public String toString()
+ {
+ return identifier + " => " + totalCount;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/twitter/src/main/java/org/apache/apex/examples/twitter/TwitterDumpApplication.java
----------------------------------------------------------------------
diff --git a/examples/twitter/src/main/java/org/apache/apex/examples/twitter/TwitterDumpApplication.java b/examples/twitter/src/main/java/org/apache/apex/examples/twitter/TwitterDumpApplication.java
new file mode 100644
index 0000000..62ec6cf
--- /dev/null
+++ b/examples/twitter/src/main/java/org/apache/apex/examples/twitter/TwitterDumpApplication.java
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.twitter;
+
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+
+import javax.annotation.Nonnull;
+
+import org.apache.hadoop.conf.Configuration;
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.DAG.Locality;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+
+import com.datatorrent.contrib.twitter.TwitterSampleInput;
+import com.datatorrent.lib.db.jdbc.AbstractJdbcTransactionableOutputOperator;
+
+import twitter4j.Status;
+
+/**
+ * An application which connects to Twitter Sample Input and stores all the
+ * tweets with their usernames in a mysql database. Please review the docs
+ * for TwitterTopCounterApplication to setup your twitter credentials. You
+ * may also be able to change JDBCStore credentials using config file.
+ *
+ * You will also need to create appropriate database and tables with the
+ * following schema, also included in mysql.sql in resources:
+ * <pre>
+ * DROP TABLE if exists tweets;
+ * CREATE TABLE tweets (
+ * window_id LONG NOT NULL,
+ * creation_date DATE,
+ * text VARCHAR(256) NOT NULL,
+ * userid VARCHAR(40) NOT NULL,
+ * KEY ( userid, creation_date)
+ * );
+ *
+ * drop table if exists dt_window_id_tracker;
+ * CREATE TABLE dt_window_id_tracker (
+ * dt_application_id VARCHAR(100) NOT NULL,
+ * dt_operator_id int(11) NOT NULL,
+ * dt_window_id bigint NOT NULL,
+ * UNIQUE (dt_application_id, dt_operator_id, dt_window_id)
+ * ) ENGINE=MyISAM DEFAULT CHARSET=latin1;
+ * </pre>
+ *
+ * @since 0.9.4
+ */
+@ApplicationAnnotation(name = "TwitterDumpExample")
+public class TwitterDumpApplication implements StreamingApplication
+{
+ public static class Status2Database extends AbstractJdbcTransactionableOutputOperator<Status>
+ {
+ public static final String INSERT_STATUS_STATEMENT = "insert into tweets (window_id, creation_date, text, userid) values (?, ?, ?, ?)";
+
+ public Status2Database()
+ {
+ store.setMetaTable("dt_window_id_tracker");
+ store.setMetaTableAppIdColumn("dt_application_id");
+ store.setMetaTableOperatorIdColumn("dt_operator_id");
+ store.setMetaTableWindowColumn("dt_window_id");
+ }
+
+ @Nonnull
+ @Override
+ protected String getUpdateCommand()
+ {
+ return INSERT_STATUS_STATEMENT;
+ }
+
+ @Override
+ protected void setStatementParameters(PreparedStatement statement, Status tuple) throws SQLException
+ {
+ statement.setLong(1, currentWindowId);
+
+ statement.setDate(2, new java.sql.Date(tuple.getCreatedAt().getTime()));
+ statement.setString(3, tuple.getText());
+ statement.setString(4, tuple.getUser().getScreenName());
+ statement.addBatch();
+ }
+ }
+
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ //dag.setAttribute(DAGContext.APPLICATION_NAME, "TweetsDump");
+
+ TwitterSampleInput twitterStream = dag.addOperator("TweetSampler", new TwitterSampleInput());
+
+ //ConsoleOutputOperator dbWriter = dag.addOperator("DatabaseWriter", new ConsoleOutputOperator());
+
+ Status2Database dbWriter = dag.addOperator("DatabaseWriter", new Status2Database());
+ dbWriter.getStore().setDatabaseDriver("com.mysql.jdbc.Driver");
+ dbWriter.getStore().setDatabaseUrl("jdbc:mysql://node6.morado.com:3306/twitter");
+ dbWriter.getStore().setConnectionProperties("user:twitter");
+
+ dag.addStream("Statuses", twitterStream.status, dbWriter.input).setLocality(Locality.CONTAINER_LOCAL);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/twitter/src/main/java/org/apache/apex/examples/twitter/TwitterDumpHBaseApplication.java
----------------------------------------------------------------------
diff --git a/examples/twitter/src/main/java/org/apache/apex/examples/twitter/TwitterDumpHBaseApplication.java b/examples/twitter/src/main/java/org/apache/apex/examples/twitter/TwitterDumpHBaseApplication.java
new file mode 100644
index 0000000..3169132
--- /dev/null
+++ b/examples/twitter/src/main/java/org/apache/apex/examples/twitter/TwitterDumpHBaseApplication.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.twitter;
+
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.Put;
+
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.DAG.Locality;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+
+import com.datatorrent.contrib.hbase.AbstractHBasePutOutputOperator;
+import com.datatorrent.contrib.twitter.TwitterSampleInput;
+
+import twitter4j.Status;
+
+/**
+ * An application which connects to Twitter Sample Input and stores all the
+ * tweets with their usernames in a hbase database. Please review the docs
+ * for TwitterTopCounterApplication to setup your twitter credentials.
+ *
+ * You need to create the HBase table to run this example. Table name can be
+ * configured but columnfamily must be 'cf' to make this example simple and complied
+ * with the mysql based example.
+ * create 'tablename', 'cf'
+ *
+ * </pre>
+ *
+ * @since 1.0.2
+ */
+@ApplicationAnnotation(name = "TwitterDumpHBaseExample")
+public class TwitterDumpHBaseApplication implements StreamingApplication
+{
+
+ public static class Status2Hbase extends AbstractHBasePutOutputOperator<Status>
+ {
+
+ @Override
+ public Put operationPut(Status t)
+ {
+ Put put = new Put(ByteBuffer.allocate(8).putLong(t.getCreatedAt().getTime()).array());
+ put.add("cf".getBytes(), "text".getBytes(), t.getText().getBytes());
+ put.add("cf".getBytes(), "userid".getBytes(), t.getText().getBytes());
+ return put;
+ }
+
+ }
+
+
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ //dag.setAttribute(DAGContext.APPLICATION_NAME, "TweetsDump");
+
+ TwitterSampleInput twitterStream = dag.addOperator("TweetSampler", new TwitterSampleInput());
+
+ Status2Hbase hBaseWriter = dag.addOperator("DatabaseWriter", new Status2Hbase());
+
+ dag.addStream("Statuses", twitterStream.status, hBaseWriter.input).setLocality(Locality.CONTAINER_LOCAL);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/twitter/src/main/java/org/apache/apex/examples/twitter/TwitterStatusHashtagExtractor.java
----------------------------------------------------------------------
diff --git a/examples/twitter/src/main/java/org/apache/apex/examples/twitter/TwitterStatusHashtagExtractor.java b/examples/twitter/src/main/java/org/apache/apex/examples/twitter/TwitterStatusHashtagExtractor.java
new file mode 100644
index 0000000..7b468ec
--- /dev/null
+++ b/examples/twitter/src/main/java/org/apache/apex/examples/twitter/TwitterStatusHashtagExtractor.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.twitter;
+
+import com.datatorrent.api.DefaultInputPort;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.common.util.BaseOperator;
+
+import twitter4j.HashtagEntity;
+import twitter4j.Status;
+
+/**
+ * <p>TwitterStatusHashtagExtractor class.</p>
+ *
+ * @since 1.0.2
+ */
+public class TwitterStatusHashtagExtractor extends BaseOperator
+{
+ public final transient DefaultOutputPort<String> hashtags = new DefaultOutputPort<String>();
+ public final transient DefaultInputPort<Status> input = new DefaultInputPort<Status>()
+ {
+ @Override
+ public void process(Status status)
+ {
+ HashtagEntity[] entities = status.getHashtagEntities();
+ if (entities != null) {
+ for (HashtagEntity he : entities) {
+ if (he != null) {
+ hashtags.emit(he.getText());
+ }
+ }
+ }
+ }
+
+ };
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/twitter/src/main/java/org/apache/apex/examples/twitter/TwitterStatusURLExtractor.java
----------------------------------------------------------------------
diff --git a/examples/twitter/src/main/java/org/apache/apex/examples/twitter/TwitterStatusURLExtractor.java b/examples/twitter/src/main/java/org/apache/apex/examples/twitter/TwitterStatusURLExtractor.java
new file mode 100644
index 0000000..40a7d3d
--- /dev/null
+++ b/examples/twitter/src/main/java/org/apache/apex/examples/twitter/TwitterStatusURLExtractor.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.twitter;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.datatorrent.api.DefaultInputPort;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.common.util.BaseOperator;
+
+import twitter4j.Status;
+import twitter4j.URLEntity;
+
+/**
+ * <p>TwitterStatusURLExtractor class.</p>
+ *
+ * @since 0.3.2
+ */
+public class TwitterStatusURLExtractor extends BaseOperator
+{
+ public final transient DefaultOutputPort<String> url = new DefaultOutputPort<String>();
+ public final transient DefaultInputPort<Status> input = new DefaultInputPort<Status>()
+ {
+ @Override
+ public void process(Status status)
+ {
+ URLEntity[] entities = status.getURLEntities();
+ if (entities != null) {
+ for (URLEntity ue: entities) {
+ if (ue != null) { // see why we intermittently get NPEs
+ url.emit((ue.getExpandedURL() == null ? ue.getURL() : ue.getExpandedURL()).toString());
+ }
+ }
+ }
+ }
+ };
+
+ private static final Logger LOG = LoggerFactory.getLogger(TwitterStatusURLExtractor.class);
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/twitter/src/main/java/org/apache/apex/examples/twitter/TwitterStatusWordExtractor.java
----------------------------------------------------------------------
diff --git a/examples/twitter/src/main/java/org/apache/apex/examples/twitter/TwitterStatusWordExtractor.java b/examples/twitter/src/main/java/org/apache/apex/examples/twitter/TwitterStatusWordExtractor.java
new file mode 100644
index 0000000..6581b76
--- /dev/null
+++ b/examples/twitter/src/main/java/org/apache/apex/examples/twitter/TwitterStatusWordExtractor.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.twitter;
+
+import java.util.Arrays;
+import java.util.HashSet;
+
+import com.datatorrent.api.Context.OperatorContext;
+import com.datatorrent.api.DefaultInputPort;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.common.util.BaseOperator;
+
+/**
+ * <p>TwitterStatusWordExtractor class.</p>
+ *
+ * @since 0.3.2
+ */
+public class TwitterStatusWordExtractor extends BaseOperator
+{
+ public HashSet<String> filterList;
+
+ public final transient DefaultOutputPort<String> output = new DefaultOutputPort<String>();
+ public final transient DefaultInputPort<String> input = new DefaultInputPort<String>()
+ {
+ @Override
+ public void process(String text)
+ {
+ String[] strs = text.split(" ");
+ if (strs != null) {
+ for (String str : strs) {
+ if (str != null && !filterList.contains(str) ) {
+ output.emit(str);
+ }
+ }
+ }
+ }
+ };
+
+ @Override
+ public void setup(OperatorContext context)
+ {
+ this.filterList = new HashSet<String>(Arrays.asList(new String[]{"", " ","I","you","the","a","to","as","he","him","his","her","she","me","can","for","of","and","or","but",
+ "this","that","!",",",".",":","#","/","@","be","in","out","was","were","is","am","are","so","no","...","my","de","RT","on","que","la","i","your","it","have","with","?","when",
+ "up","just","do","at","&","-","+","*","\\","y","n","like","se","en","te","el","I'm"}));
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/twitter/src/main/java/org/apache/apex/examples/twitter/TwitterTopCounterApplication.java
----------------------------------------------------------------------
diff --git a/examples/twitter/src/main/java/org/apache/apex/examples/twitter/TwitterTopCounterApplication.java b/examples/twitter/src/main/java/org/apache/apex/examples/twitter/TwitterTopCounterApplication.java
new file mode 100644
index 0000000..ee43383
--- /dev/null
+++ b/examples/twitter/src/main/java/org/apache/apex/examples/twitter/TwitterTopCounterApplication.java
@@ -0,0 +1,222 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.twitter;
+
+import java.net.URI;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+
+import com.google.common.collect.Maps;
+
+import com.datatorrent.api.Context;
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.DAG.Locality;
+import com.datatorrent.api.Operator;
+import com.datatorrent.api.Operator.OutputPort;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+
+import com.datatorrent.contrib.twitter.TwitterSampleInput;
+import com.datatorrent.lib.algo.UniqueCounter;
+import com.datatorrent.lib.appdata.schemas.SchemaUtils;
+import com.datatorrent.lib.appdata.snapshot.AppDataSnapshotServerMap;
+import com.datatorrent.lib.io.ConsoleOutputOperator;
+import com.datatorrent.lib.io.PubSubWebSocketAppDataQuery;
+import com.datatorrent.lib.io.PubSubWebSocketAppDataResult;
+
+/**
+ * Twitter Example Application: <br>
+ * This example application samples random public status from twitter, send to url
+ * extractor. <br>
+ * Top 10 url(s) mentioned in tweets in last 5 mins are displayed on every
+ * window count (500ms).<br>
+ * <br>
+ *
+ * Real Time Calculation :<br>
+ * This application calculates top 10 url mentioned in tweets in last 5
+ * minutes across a 1% random tweet sampling on a rolling window basis.<br>
+ * <br>
+ * Before running this application, you need to have a <a href="https://dev.twitter.com/apps">Twitter API account</a>
+ * and configure the authentication. For launch from CLI, those go into ~/.dt/dt-site.xml:
+ * <pre>
+ * {@code
+ * <?xml version="1.0" encoding="UTF-8"?>
+ * <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+ * <configuration>
+ *
+ * <property> <name>dt.operator.TweetSampler.consumerKey</name>
+ * <value>TBD</value> </property>
+ *
+ * <property> <name>dt.operator.TweetSampler.consumerSecret</name>
+ * <value>TBD</value> </property>
+ *
+ * <property> <name>dt.operator.TweetSampler.accessToken</name>
+ * <value>TBD</value> </property>
+ *
+ * <property> <name>dt.operator.TweetSampler.accessTokenSecret</name>
+ * <value>TBD</value> </property>
+ * </configuration>
+ * }
+ * </pre>
+ * Custom Attributes: <br>
+ * <b>topCounts operator : <b>
+ * <ul>
+ * <li>Top Count : 10, number of top unique url to be reported.</li>
+ * <li>Sliding window count : 600, report over last 5 min (600 * .5 / 60 mins)</li>
+ * <li>window slide value : 1</li>
+ * </ul>
+ * <p>
+ * Running Java Test or Main app in IDE:
+ *
+ * <pre>
+ * LocalMode.runApp(new Application(), 600000); // 10 min run
+ * </pre>
+ *
+ * Run Success : <br>
+ * For successful deployment and run, user should see following output on
+ * console:
+ *
+ * <pre>
+ * topURLs: {http://goo.gl/V0R05=2, http://etsy.me/10r1Yg3=6, http://tinyurl.com/88b5jqb=2, http://www.justunfollow.com=4, http://fllwrs.com=2, http://goo.gl/a9Sjp=2, http://goo.gl/iKeVH=2, http://Unfollowers.me=7, http://freetexthost.com/j3y03la4g3=2, http://uranaitter.com=4}
+ * topURLs: {http://goo.gl/V0R05=2, http://etsy.me/10r1Yg3=6, http://tinyurl.com/88b5jqb=2, http://www.justunfollow.com=4, http://fllwrs.com=2, http://goo.gl/a9Sjp=2, http://goo.gl/iKeVH=2, http://Unfollowers.me=7, http://freetexthost.com/j3y03la4g3=2, http://uranaitter.com=4}
+ * topURLs: {http://goo.gl/V0R05=2, http://etsy.me/10r1Yg3=6, http://tinyurl.com/88b5jqb=2, http://www.justunfollow.com=4, http://fllwrs.com=2, http://goo.gl/a9Sjp=2, http://goo.gl/iKeVH=2, http://Unfollowers.me=7, http://freetexthost.com/j3y03la4g3=2, http://uranaitter.com=4}
+ * topURLs: {http://goo.gl/V0R05=2, http://etsy.me/10r1Yg3=6, http://tinyurl.com/88b5jqb=2, http://www.justunfollow.com=4, http://fllwrs.com=2, http://goo.gl/a9Sjp=2, http://goo.gl/iKeVH=2, http://Unfollowers.me=7, http://freetexthost.com/j3y03la4g3=2, http://uranaitter.com=4}
+ * topURLs: {http://goo.gl/V0R05=2, http://etsy.me/10r1Yg3=6, http://tinyurl.com/88b5jqb=2, http://www.justunfollow.com=4, http://fllwrs.com=2, http://goo.gl/a9Sjp=2, http://goo.gl/iKeVH=2, http://Unfollowers.me=7, http://freetexthost.com/j3y03la4g3=2, http://uranaitter.com=4}
+ * topURLs: {http://goo.gl/V0R05=2, http://etsy.me/10r1Yg3=6, http://tinyurl.com/88b5jqb=2, http://www.justunfollow.com=4, http://fllwrs.com=2, http://goo.gl/a9Sjp=2, http://goo.gl/iKeVH=2, http://Unfollowers.me=7, http://freetexthost.com/j3y03la4g3=2, http://uranaitter.com=4}
+ * topURLs: {http://goo.gl/V0R05=2, http://etsy.me/10r1Yg3=6, http://tinyurl.com/88b5jqb=2, http://www.justunfollow.com=4, http://fllwrs.com=2, http://goo.gl/a9Sjp=2, http://goo.gl/iKeVH=2, http://Unfollowers.me=7, http://freetexthost.com/j3y03la4g3=2, http://uranaitter.com=4}
+ * 2013-06-17 14:38:55,201 [main] INFO stram.StramLocalCluster run - Application finished.
+ * 2013-06-17 14:38:55,201 [container-2] INFO stram.StramChild processHeartbeatResponse - Received shutdown request
+ * </pre>
+ *
+ * Scaling Options : <br>
+ * User can scale application by setting intial partition size > 1 on count
+ * unique operator. <br>
+ * <br>
+ *
+ * Application DAG : <br>
+ * <img src="doc-files/Application.gif" width=600px > <br>
+ * <br>
+ *
+ * Streaming Window Size : 500ms(default) <br>
+ * Operator Details : <br>
+ * <ul>
+ * <li><b>The twitterFeed operator : </b> This operator samples random public
+ * statues from twitter and emits to application. <br>
+ * Class : com.datatorrent.examples.twitter.TwitterSampleInput <br>
+ * StateFull : No, window count 1 <br>
+ * </li>
+ * <li><b>The urlExtractor operator : </b> This operator extracts url from
+ * random sampled statues from twitter. <br>
+ * Class : {@link TwitterStatusURLExtractor} <br>
+ * StateFull : No, window count 1 <br>
+ * </li>
+ * <li><b>The uniqueCounter operator : </b> This operator aggregates count for each
+ * url extracted from random samples. <br>
+ * Class : {@link com.datatorrent.lib.algo.UniqueCounter} <br>
+ * StateFull : No, window count 1 <br>
+ * </li>
+ * <li><b> The topCounts operator : </b> This operator caluculates top url in last 1
+ * min sliding window count 1. <br>
+ * Class : com.datatorrent.lib.algo.WindowedTopCounter <br>
+ * StateFull : Yes, sliding window count 120 (1 min) <br>
+ * </li>
+ * <li><b>The operator Console: </b> This operator just outputs the input tuples
+ * to the console (or stdout). <br>
+ * </li>
+ * </ul>
+ *
+ * @since 0.3.2
+ */
+@ApplicationAnnotation(name = TwitterTopCounterApplication.APP_NAME)
+public class TwitterTopCounterApplication implements StreamingApplication
+{
+ public static final String SNAPSHOT_SCHEMA = "twitterURLDataSchema.json";
+ public static final String CONVERSION_SCHEMA = "twitterURLConverterSchema.json";
+ public static final String APP_NAME = "TwitterExample";
+
+ private final Locality locality = null;
+
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ // Setup the operator to get the data from twitter sample stream injected into the system.
+ TwitterSampleInput twitterFeed = new TwitterSampleInput();
+ twitterFeed = dag.addOperator("TweetSampler", twitterFeed);
+
+ // Setup the operator to get the URLs extracted from the twitter statuses
+ TwitterStatusURLExtractor urlExtractor = dag.addOperator("URLExtractor", TwitterStatusURLExtractor.class);
+
+ // Setup a node to count the unique urls within a window.
+ UniqueCounter<String> uniqueCounter = dag.addOperator("UniqueURLCounter", new UniqueCounter<String>());
+ // Get the aggregated url counts and count them over last 5 mins.
+ dag.setAttribute(uniqueCounter, Context.OperatorContext.APPLICATION_WINDOW_COUNT, 600);
+ dag.setAttribute(uniqueCounter, Context.OperatorContext.SLIDE_BY_WINDOW_COUNT, 1);
+
+
+ WindowedTopCounter<String> topCounts = dag.addOperator("TopCounter", new WindowedTopCounter<String>());
+ topCounts.setTopCount(10);
+ topCounts.setSlidingWindowWidth(1);
+ topCounts.setDagWindowWidth(1);
+
+ // Feed the statuses from feed into the input of the url extractor.
+ dag.addStream("TweetStream", twitterFeed.status, urlExtractor.input).setLocality(Locality.CONTAINER_LOCAL);
+ // Start counting the urls coming out of URL extractor
+ dag.addStream("TwittedURLs", urlExtractor.url, uniqueCounter.data).setLocality(locality);
+ // Count unique urls
+ dag.addStream("UniqueURLCounts", uniqueCounter.count, topCounts.input);
+
+ consoleOutput(dag, "topURLs", topCounts.output, SNAPSHOT_SCHEMA, "url");
+ }
+
+ public static void consoleOutput(DAG dag, String operatorName, OutputPort<List<Map<String, Object>>> topCount, String schemaFile, String alias)
+ {
+ String gatewayAddress = dag.getValue(DAG.GATEWAY_CONNECT_ADDRESS);
+ if (!StringUtils.isEmpty(gatewayAddress)) {
+ URI uri = URI.create("ws://" + gatewayAddress + "/pubsub");
+
+ AppDataSnapshotServerMap snapshotServer = dag.addOperator("SnapshotServer", new AppDataSnapshotServerMap());
+
+ Map<String, String> conversionMap = Maps.newHashMap();
+ conversionMap.put(alias, WindowedTopCounter.FIELD_TYPE);
+ String snapshotServerJSON = SchemaUtils.jarResourceFileToString(schemaFile);
+
+ snapshotServer.setSnapshotSchemaJSON(snapshotServerJSON);
+ snapshotServer.setTableFieldToMapField(conversionMap);
+
+ PubSubWebSocketAppDataQuery wsQuery = new PubSubWebSocketAppDataQuery();
+ wsQuery.setUri(uri);
+ snapshotServer.setEmbeddableQueryInfoProvider(wsQuery);
+
+ PubSubWebSocketAppDataResult wsResult = dag.addOperator("QueryResult", new PubSubWebSocketAppDataResult());
+ wsResult.setUri(uri);
+ Operator.InputPort<String> queryResultPort = wsResult.input;
+
+ dag.addStream("MapProvider", topCount, snapshotServer.input);
+ dag.addStream("Result", snapshotServer.queryResult, queryResultPort);
+ } else {
+ ConsoleOutputOperator operator = dag.addOperator(operatorName, new ConsoleOutputOperator());
+ operator.setStringFormat(operatorName + ": %s");
+
+ dag.addStream("MapProvider", topCount, operator.input);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/twitter/src/main/java/org/apache/apex/examples/twitter/TwitterTopWordsApplication.java
----------------------------------------------------------------------
diff --git a/examples/twitter/src/main/java/org/apache/apex/examples/twitter/TwitterTopWordsApplication.java b/examples/twitter/src/main/java/org/apache/apex/examples/twitter/TwitterTopWordsApplication.java
new file mode 100644
index 0000000..fb274ea
--- /dev/null
+++ b/examples/twitter/src/main/java/org/apache/apex/examples/twitter/TwitterTopWordsApplication.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.twitter;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.DAG.Locality;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.contrib.twitter.TwitterSampleInput;
+import com.datatorrent.lib.algo.UniqueCounter;
+
+/**
+ * This application is same as other twitter example
+ * {@link TwitterTopCounterApplication} <br>
+ * Run Sample :
+ *
+ * <pre>
+ * 2013-06-17 16:50:34,911 [Twitter Stream consumer-1[Establishing connection]] INFO twitter4j.TwitterStreamImpl info - Connection established.
+ * 2013-06-17 16:50:34,912 [Twitter Stream consumer-1[Establishing connection]] INFO twitter4j.TwitterStreamImpl info - Receiving status stream.
+ * topWords: {}
+ * topWords: {love=1, ate=1, catch=1, calma=1, Phillies=1, ela=1, from=1, running=1}
+ * </pre>
+ *
+ * @since 0.3.2
+ */
+@ApplicationAnnotation(name = TwitterTopWordsApplication.APP_NAME)
+public class TwitterTopWordsApplication implements StreamingApplication
+{
+ public static final String SNAPSHOT_SCHEMA = "twitterWordDataSchema.json";
+ public static final String CONVERSION_SCHEMA = "twitterWordConverterSchema.json";
+ public static final String APP_NAME = "RollingTopWordsExample";
+ public static final String PROP_USE_APPDATA = "dt.application." + APP_NAME + ".useAppData";
+
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ TwitterSampleInput twitterFeed = new TwitterSampleInput();
+ twitterFeed = dag.addOperator("TweetSampler", twitterFeed);
+
+ TwitterStatusWordExtractor wordExtractor = dag.addOperator("WordExtractor", TwitterStatusWordExtractor.class);
+ UniqueCounter<String> uniqueCounter = dag.addOperator("UniqueWordCounter", new UniqueCounter<String>());
+ WindowedTopCounter<String> topCounts = dag.addOperator("TopCounter", new WindowedTopCounter<String>());
+
+ topCounts.setSlidingWindowWidth(120);
+ topCounts.setDagWindowWidth(1);
+
+ dag.addStream("TweetStream", twitterFeed.text, wordExtractor.input);
+ dag.addStream("TwittedWords", wordExtractor.output, uniqueCounter.data);
+ dag.addStream("UniqueWordCounts", uniqueCounter.count, topCounts.input).setLocality(Locality.CONTAINER_LOCAL);
+
+ TwitterTopCounterApplication.consoleOutput(dag, "topWords", topCounts.output, SNAPSHOT_SCHEMA, "word");
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/twitter/src/main/java/org/apache/apex/examples/twitter/TwitterTrendingHashtagsApplication.java
----------------------------------------------------------------------
diff --git a/examples/twitter/src/main/java/org/apache/apex/examples/twitter/TwitterTrendingHashtagsApplication.java b/examples/twitter/src/main/java/org/apache/apex/examples/twitter/TwitterTrendingHashtagsApplication.java
new file mode 100644
index 0000000..7a03a64
--- /dev/null
+++ b/examples/twitter/src/main/java/org/apache/apex/examples/twitter/TwitterTrendingHashtagsApplication.java
@@ -0,0 +1,166 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.twitter;
+
+import org.apache.hadoop.conf.Configuration;
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.DAG.Locality;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.contrib.twitter.TwitterSampleInput;
+import com.datatorrent.lib.algo.UniqueCounter;
+
+/**
+ * Twitter Example Application: <br>
+ * This example application samples random public status from twitter, send to Hashtag
+ * extractor. <br>
+ * Top 10 Hashtag(s) mentioned in tweets in last 5 mins are displayed on every
+ * window count (500ms).<br>
+ * <br>
+ *
+ * Real Time Calculation :<br>
+ * This application calculates top 10 Hashtag mentioned in tweets in last 5
+ * minutes across a 1% random tweet sampling on a rolling window basis.<br>
+ * <br>
+ * Before running this application, you need to have a <a href="https://dev.twitter.com/apps">Twitter API account</a>
+ * and configure the authentication. For launch from CLI, those go into ~/.dt/dt-site.xml:
+ * <pre>
+ * {@code
+ * <?xml version="1.0" encoding="UTF-8"?>
+ * <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+ * <configuration>
+ *
+ * <property> <name>dt.operator.TweetSampler.prop.consumerKey</name>
+ * <value>TBD</value> </property>
+ *
+ * <property> <name>dt.operator.TweetSampler.prop.consumerSecret</name>
+ * <value>TBD</value> </property>
+ *
+ * <property> <name>dt.operator.TweetSampler.prop.accessToken</name>
+ * <value>TBD</value> </property>
+ *
+ * <property> <name>dt.operator.TweetSampler.prop.accessTokenSecret</name>
+ * <value>TBD</value> </property>
+ * </configuration>
+ * }
+ * </pre>
+ * Custom Attributes: <br>
+ * <b>topCounts operator : <b>
+ * <ul>
+ * <li>Top Count : 10, number of top unique Hashtag to be reported.</li>
+ * <li>Sliding window count : 600, report over last 5 min (600 * .5 / 60 mins)</li>
+ * <li>window slide value : 1</li>
+ * </ul>
+ * <p>
+ * Running Java Test or Main app in IDE:
+ *
+ * <pre>
+ * LocalMode.runApp(new Application(), 600000); // 10 min run
+ * </pre>
+ *
+ * Run Success : <br>
+ * For successful deployment and run, user should see similar output on console as below:
+ *
+ * <pre>
+ * topHashtags: {"string": "{\"gameinsight\":\"12\",\"android\":\"8\",\"WotaJKT48alay\":\"12\",\"TernyataJKT48CyberItuForoumSampah\":\"8\",\"NHL15Subban\":\"30\",\"JKT48CYBERKeracunanCUKRIK\":\"8\",\"verifydjzoodel\":\"59\",\"androidgames\":\"9\",\"\u0645\u0639_\u0627\u0644\u0644\u0647\":\"10\",\"Jct48cyberTAIBABI\":\"10\"}"}
+ * topHashtags: {"string": "{\"gameinsight\":\"12\",\"android\":\"8\",\"WotaJKT48alay\":\"12\",\"TernyataJKT48CyberItuForoumSampah\":\"8\",\"NHL15Subban\":\"30\",\"JKT48CYBERKeracunanCUKRIK\":\"8\",\"verifydjzoodel\":\"59\",\"androidgames\":\"9\",\"\u0645\u0639_\u0627\u0644\u0644\u0647\":\"10\",\"Jct48cyberTAIBABI\":\"10\"}"}
+ * topHashtags: {"string": "{\"gameinsight\":\"12\",\"android\":\"8\",\"WotaJKT48alay\":\"12\",\"TernyataJKT48CyberItuForoumSampah\":\"8\",\"NHL15Subban\":\"30\",\"JKT48CYBERKeracunanCUKRIK\":\"8\",\"verifydjzoodel\":\"59\",\"androidgames\":\"9\",\"\u0645\u0639_\u0627\u0644\u0644\u0647\":\"10\",\"Jct48cyberTAIBABI\":\"10\"}"}
+ * topHashtags: {"string": "{\"gameinsight\":\"12\",\"android\":\"8\",\"WotaJKT48alay\":\"12\",\"TernyataJKT48CyberItuForoumSampah\":\"8\",\"NHL15Subban\":\"30\",\"JKT48CYBERKeracunanCUKRIK\":\"8\",\"verifydjzoodel\":\"59\",\"androidgames\":\"9\",\"\u0645\u0639_\u0627\u0644\u0644\u0647\":\"10\",\"Jct48cyberTAIBABI\":\"10\"}"}
+ * topHashtags: {"string": "{\"gameinsight\":\"12\",\"android\":\"8\",\"WotaJKT48alay\":\"12\",\"TernyataJKT48CyberItuForoumSampah\":\"8\",\"NHL15Subban\":\"30\",\"JKT48CYBERKeracunanCUKRIK\":\"8\",\"verifydjzoodel\":\"59\",\"androidgames\":\"9\",\"\u0645\u0639_\u0627\u0644\u0644\u0647\":\"10\",\"Jct48cyberTAIBABI\":\"10\"}"}
+ * topHashtags: {"string": "{\"gameinsight\":\"12\",\"android\":\"8\",\"WotaJKT48alay\":\"12\",\"TernyataJKT48CyberItuForoumSampah\":\"8\",\"NHL15Subban\":\"30\",\"JKT48CYBERKeracunanCUKRIK\":\"8\",\"verifydjzoodel\":\"59\",\"androidgames\":\"9\",\"\u0645\u0639_\u0627\u0644\u0644\u0647\":\"10\",\"Jct48cyberTAIBABI\":\"10\"}"}
+ * topHashtags: {"string": "{\"gameinsight\":\"12\",\"android\":\"8\",\"WotaJKT48alay\":\"12\",\"TernyataJKT48CyberItuForoumSampah\":\"8\",\"NHL15Subban\":\"30\",\"JKT48CYBERKeracunanCUKRIK\":\"8\",\"verifydjzoodel\":\"59\",\"androidgames\":\"9\",\"\u0645\u0639_\u0627\u0644\u0644\u0647\":\"10\",\"Jct48cyberTAIBABI\":\"10\"}"}
+ * topHashtags: {"string": "{\"gameinsight\":\"12\",\"android\":\"8\",\"WotaJKT48alay\":\"12\",\"TernyataJKT48CyberItuForoumSampah\":\"8\",\"NHL15Subban\":\"30\",\"JKT48CYBERKeracunanCUKRIK\":\"8\",\"verifydjzoodel\":\"59\",\"androidgames\":\"9\",\"\u0645\u0639_\u0627\u0644\u0644\u0647\":\"10\",\"Jct48cyberTAIBABI\":\"10\"}"}
+ * topHashtags: {"string": "{\"gameinsight\":\"12\",\"android\":\"8\",\"WotaJKT48alay\":\"12\",\"TernyataJKT48CyberItuForoumSampah\":\"8\",\"NHL15Subban\":\"30\",\"JKT48CYBERKeracunanCUKRIK\":\"8\",\"verifydjzoodel\":\"59\",\"androidgames\":\"9\",\"\u0645\u0639_\u0627\u0644\u0644\u0647\":\"10\",\"Jct48cyberTAIBABI\":\"10\"}"}
+ * topHashtags: {"string": "{\"gameinsight\":\"12\",\"android\":\"8\",\"WotaJKT48alay\":\"12\",\"TernyataJKT48CyberItuForoumSampah\":\"8\",\"NHL15Subban\":\"30\",\"JKT48CYBERKeracunanCUKRIK\":\"8\",\"verifydjzoodel\":\"59\",\"androidgames\":\"9\",\"\u0645\u0639_\u0627\u0644\u0644\u0647\":\"10\",\"Jct48cyberTAIBABI\":\"10\"}"}
+ * 2013-06-17 14:38:55,201 [main] INFO stram.StramLocalCluster run - Application finished.
+ * 2013-06-17 14:38:55,201 [container-2] INFO stram.StramChild processHeartbeatResponse - Received shutdown request
+ * </pre>
+ *
+ * Scaling Options : <br>
+ * User can scale application by setting intial partition size > 1 on count
+ * unique operator. <br>
+ * <br>
+ *
+ * Application DAG : <br>
+ * <img src="doc-files/Application.gif" width=600px > <br>
+ * <br>
+ *
+ * Streaming Window Size : 500ms(default) <br>
+ * Operator Details : <br>
+ * <ul>
+ * <li><b>The twitterFeed operator : </b> This operator samples random public
+ * statues from twitter and emits to application. <br>
+ * Class : com.datatorrent.examples.twitter.TwitterSampleInput <br>
+ * StateFull : No, window count 1 <br>
+ * </li>
+ * <li><b>The HashtagExtractor operator : </b> This operator extracts Hashtag from
+ * random sampled statues from twitter. <br>
+ * Class : {@link TwitterStatusHashtagExtractor} <br>
+ * StateFull : No, window count 1 <br>
+ * </li>
+ * <li><b>The uniqueCounter operator : </b> This operator aggregates count for each
+ * Hashtag extracted from random samples. <br>
+ * Class : {@link com.datatorrent.lib.algo.UniqueCounter} <br>
+ * StateFull : No, window count 1 <br>
+ * </li>
+ * <li><b> The topCounts operator : </b> This operator caluculates top Hashtag in last 1
+ * min sliding window count 1. <br>
+ * Class : com.datatorrent.lib.algo.WindowedTopCounter <br>
+ * StateFull : Yes, sliding window count 120 (1 min) <br>
+ * </li>
+ * <li><b>The operator Console: </b> This operator just outputs the input tuples
+ * to the console (or stdout). <br>
+ * </li>
+ * </ul>
+ *
+ * @since 1.0.2
+ */
+@ApplicationAnnotation(name = TwitterTrendingHashtagsApplication.APP_NAME)
+public class TwitterTrendingHashtagsApplication implements StreamingApplication
+{
+ public static final String SNAPSHOT_SCHEMA = "twitterHashTagDataSchema.json";
+ public static final String CONVERSION_SCHEMA = "twitterHashTagConverterSchema.json";
+ public static final String APP_NAME = "TwitterTrendingExample";
+ public static final String PROP_USE_APPDATA = "dt.application." + APP_NAME + ".useAppData";
+
+ private final Locality locality = null;
+
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ // Setup the operator to get the data from twitter sample stream injected into the system.
+ TwitterSampleInput twitterFeed = new TwitterSampleInput();
+ twitterFeed = dag.addOperator("TweetSampler", twitterFeed);
+
+ // Setup a node to count the unique Hashtags within a window.
+ UniqueCounter<String> uniqueCounter = dag.addOperator("UniqueHashtagCounter", new UniqueCounter<String>());
+
+ // Get the aggregated Hashtag counts and count them over last 5 mins.
+ WindowedTopCounter<String> topCounts = dag.addOperator("TopCounter", new WindowedTopCounter<String>());
+ topCounts.setTopCount(10);
+ topCounts.setSlidingWindowWidth(600);
+ topCounts.setDagWindowWidth(1);
+
+ dag.addStream("TwittedHashtags", twitterFeed.hashtag, uniqueCounter.data).setLocality(locality);
+ // Count unique Hashtags
+ dag.addStream("UniqueHashtagCounts", uniqueCounter.count, topCounts.input);
+
+ TwitterTopCounterApplication.consoleOutput(dag, "topHashtags", topCounts.output, SNAPSHOT_SCHEMA, "hashtag");
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/twitter/src/main/java/org/apache/apex/examples/twitter/URLSerDe.java
----------------------------------------------------------------------
diff --git a/examples/twitter/src/main/java/org/apache/apex/examples/twitter/URLSerDe.java b/examples/twitter/src/main/java/org/apache/apex/examples/twitter/URLSerDe.java
new file mode 100644
index 0000000..0c3f481
--- /dev/null
+++ b/examples/twitter/src/main/java/org/apache/apex/examples/twitter/URLSerDe.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.twitter;
+
+import java.nio.ByteBuffer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import com.datatorrent.api.StreamCodec;
+import com.datatorrent.netlet.util.Slice;
+
+/**
+ * <p>URLSerDe class.</p>
+ *
+ * @since 0.3.2
+ */
+public class URLSerDe implements StreamCodec<byte[]>
+{
+ /**
+ * Covert the bytes into object useful for downstream node.
+ *
+ * @param fragment
+ * @return WindowedURLHolder object which represents the bytes.
+ */
+ @Override
+ public byte[] fromByteArray(Slice fragment)
+ {
+ if (fragment == null || fragment.buffer == null) {
+ return null;
+ } else if (fragment.offset == 0 && fragment.length == fragment.buffer.length) {
+ return fragment.buffer;
+ } else {
+ byte[] buffer = new byte[fragment.buffer.length];
+ System.arraycopy(fragment.buffer, fragment.offset, buffer, 0, fragment.length);
+ return buffer;
+ }
+ }
+
+ /**
+ * Cast the input object to byte[].
+ *
+ * @param object - byte array representing the bytes of the string
+ * @return the same object as input
+ */
+ @Override
+ public Slice toByteArray(byte[] object)
+ {
+ return new Slice(object, 0, object.length);
+ }
+
+ @Override
+ public int getPartition(byte[] object)
+ {
+ ByteBuffer bb = ByteBuffer.wrap(object);
+ return bb.hashCode();
+ }
+
+ private static final Logger logger = LoggerFactory.getLogger(URLSerDe.class);
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/twitter/src/main/java/org/apache/apex/examples/twitter/WindowedTopCounter.java
----------------------------------------------------------------------
diff --git a/examples/twitter/src/main/java/org/apache/apex/examples/twitter/WindowedTopCounter.java b/examples/twitter/src/main/java/org/apache/apex/examples/twitter/WindowedTopCounter.java
new file mode 100644
index 0000000..8ecd6ae
--- /dev/null
+++ b/examples/twitter/src/main/java/org/apache/apex/examples/twitter/WindowedTopCounter.java
@@ -0,0 +1,282 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.apex.examples.twitter;
+
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.PriorityQueue;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+import com.datatorrent.api.Context.OperatorContext;
+import com.datatorrent.api.DefaultInputPort;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.common.util.BaseOperator;
+
+/**
+ *
+ * WindowedTopCounter is an operator which counts the most often occurring tuples in a sliding window of a specific size.
+ * The operator expects to receive a map object which contains a set of objects mapped to their respective frequency of
+ * occurrences. e.g. if we are looking at most commonly occurring names then the operator expects to receive the tuples
+ * of type Map<String, Intenger> on its input port, and at the end of the window it emits 1 object of type Map<String, Integer>
+ * with a pre determined size. The emitted object contains the most frequently occurring keys.
+ *
+ * @param <T> Type of the key in the map object which is accepted on input port as payload. Note that this key must be HashMap friendly.
+ * @since 0.3.2
+ */
+public class WindowedTopCounter<T> extends BaseOperator
+{
+ public static final String FIELD_TYPE = "type";
+ public static final String FIELD_COUNT = "count";
+
+ private static final Logger logger = LoggerFactory.getLogger(WindowedTopCounter.class);
+
+ private PriorityQueue<SlidingContainer<T>> topCounter;
+ private int windows;
+ private int topCount = 10;
+ private int slidingWindowWidth;
+ private int dagWindowWidth;
+ private HashMap<T, SlidingContainer<T>> objects = new HashMap<T, SlidingContainer<T>>();
+
+ /**
+ * Input port on which map objects containing keys with their respective frequency as values will be accepted.
+ */
+ public final transient DefaultInputPort<Map<T, Integer>> input = new DefaultInputPort<Map<T, Integer>>()
+ {
+ @Override
+ public void process(Map<T, Integer> map)
+ {
+ for (Map.Entry<T, Integer> e : map.entrySet()) {
+ SlidingContainer<T> holder = objects.get(e.getKey());
+ if (holder == null) {
+ holder = new SlidingContainer<T>(e.getKey(), windows);
+ objects.put(e.getKey(), holder);
+ }
+ holder.adjustCount(e.getValue());
+ }
+ }
+ };
+
+ public final transient DefaultOutputPort<List<Map<String, Object>>> output = new DefaultOutputPort<List<Map<String, Object>>>();
+
+ @Override
+ public void setup(OperatorContext context)
+ {
+ windows = (int)(slidingWindowWidth / dagWindowWidth) + 1;
+ if (slidingWindowWidth % dagWindowWidth != 0) {
+ logger.warn("slidingWindowWidth(" + slidingWindowWidth + ") is not exact multiple of dagWindowWidth(" + dagWindowWidth + ")");
+ }
+
+ topCounter = new PriorityQueue<SlidingContainer<T>>(this.topCount, new TopSpotComparator());
+ }
+
+ @Override
+ public void beginWindow(long windowId)
+ {
+ topCounter.clear();
+ }
+
+ @Override
+ public void endWindow()
+ {
+ Iterator<Map.Entry<T, SlidingContainer<T>>> iterator = objects.entrySet().iterator();
+ int i = topCount;
+
+ /*
+ * Try to fill the priority queue with the first topCount URLs.
+ */
+ SlidingContainer<T> holder;
+ while (iterator.hasNext()) {
+ holder = iterator.next().getValue();
+ holder.slide();
+
+ if (holder.totalCount == 0) {
+ iterator.remove();
+ } else {
+ topCounter.add(holder);
+ if (--i == 0) {
+ break;
+ }
+ }
+ }
+ logger.debug("objects.size(): {}", objects.size());
+
+ /*
+ * Make room for the new element in the priority queue by deleting the
+ * smallest one, if we KNOW that the new element is useful to us.
+ */
+ if (i == 0) {
+ int smallest = topCounter.peek().totalCount;
+ while (iterator.hasNext()) {
+ holder = iterator.next().getValue();
+ holder.slide();
+
+ if (holder.totalCount > smallest) {
+ topCounter.poll();
+ topCounter.add(holder);
+ smallest = topCounter.peek().totalCount;
+ } else if (holder.totalCount == 0) {
+ iterator.remove();
+ }
+ }
+ }
+
+ List<Map<String, Object>> data = Lists.newArrayList();
+
+ Iterator<SlidingContainer<T>> topIter = topCounter.iterator();
+
+ while (topIter.hasNext()) {
+ final SlidingContainer<T> wh = topIter.next();
+ Map<String, Object> tableRow = Maps.newHashMap();
+
+ tableRow.put(FIELD_TYPE, wh.identifier.toString());
+ tableRow.put(FIELD_COUNT, wh.totalCount);
+
+ data.add(tableRow);
+ }
+
+ Collections.sort(data, TwitterOutputSorter.INSTANCE);
+
+ output.emit(data);
+ topCounter.clear();
+ }
+
+ @Override
+ public void teardown()
+ {
+ topCounter = null;
+ objects = null;
+ }
+
+ /**
+ * Set the count of most frequently occurring keys to emit per map object.
+ *
+ * @param count count of the objects in the map emitted at the output port.
+ */
+ public void setTopCount(int count)
+ {
+ topCount = count;
+ }
+
+ public int getTopCount()
+ {
+ return topCount;
+ }
+
+ /**
+ * @return the windows
+ */
+ public int getWindows()
+ {
+ return windows;
+ }
+
+ /**
+ * @param windows the windows to set
+ */
+ public void setWindows(int windows)
+ {
+ this.windows = windows;
+ }
+
+ /**
+ * @return the slidingWindowWidth
+ */
+ public int getSlidingWindowWidth()
+ {
+ return slidingWindowWidth;
+ }
+
+ /**
+ * Set the width of the sliding window.
+ *
+ * Sliding window is typically much larger than the dag window. e.g. One may want to measure the most frequently
+ * occurring keys over the period of 5 minutes. So if dagWindowWidth (which is by default 500ms) is set to 500ms,
+ * the slidingWindowWidth would be (60 * 5 * 1000 =) 300000.
+ *
+ * @param slidingWindowWidth - Sliding window width to be set for this operator, recommended to be multiple of DAG window.
+ */
+ public void setSlidingWindowWidth(int slidingWindowWidth)
+ {
+ this.slidingWindowWidth = slidingWindowWidth;
+ }
+
+ /**
+ * @return the dagWindowWidth
+ */
+ public int getDagWindowWidth()
+ {
+ return dagWindowWidth;
+ }
+
+ /**
+ * Set the width of the sliding window.
+ *
+ * Sliding window is typically much larger than the dag window. e.g. One may want to measure the most frequently
+ * occurring keys over the period of 5 minutes. So if dagWindowWidth (which is by default 500ms) is set to 500ms,
+ * the slidingWindowWidth would be (60 * 5 * 1000 =) 300000.
+ *
+ * @param dagWindowWidth - DAG's native window width. It has to be the value of the native window set at the application level.
+ */
+ public void setDagWindowWidth(int dagWindowWidth)
+ {
+ this.dagWindowWidth = dagWindowWidth;
+ }
+
+ static class TopSpotComparator implements Comparator<SlidingContainer<?>>
+ {
+ @Override
+ public int compare(SlidingContainer<?> o1, SlidingContainer<?> o2)
+ {
+ if (o1.totalCount > o2.totalCount) {
+ return 1;
+ } else if (o1.totalCount < o2.totalCount) {
+ return -1;
+ }
+
+ return 0;
+ }
+ }
+
+ private static class TwitterOutputSorter implements Comparator<Map<String, Object>>
+ {
+ public static final TwitterOutputSorter INSTANCE = new TwitterOutputSorter();
+
+ private TwitterOutputSorter()
+ {
+ }
+
+ @Override
+ public int compare(Map<String, Object> o1, Map<String, Object> o2)
+ {
+ Integer count1 = (Integer)o1.get(FIELD_COUNT);
+ Integer count2 = (Integer)o2.get(FIELD_COUNT);
+
+ return count1.compareTo(count2);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/twitter/src/main/java/org/apache/apex/examples/twitter/doc-files/Application.gif
----------------------------------------------------------------------
diff --git a/examples/twitter/src/main/java/org/apache/apex/examples/twitter/doc-files/Application.gif b/examples/twitter/src/main/java/org/apache/apex/examples/twitter/doc-files/Application.gif
new file mode 100644
index 0000000..d21e1d9
Binary files /dev/null and b/examples/twitter/src/main/java/org/apache/apex/examples/twitter/doc-files/Application.gif differ
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/twitter/src/main/java/org/apache/apex/examples/twitter/package-info.java
----------------------------------------------------------------------
diff --git a/examples/twitter/src/main/java/org/apache/apex/examples/twitter/package-info.java b/examples/twitter/src/main/java/org/apache/apex/examples/twitter/package-info.java
new file mode 100644
index 0000000..956ff08
--- /dev/null
+++ b/examples/twitter/src/main/java/org/apache/apex/examples/twitter/package-info.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/**
+ * Twitter top URL's demonstration application.
+ */
+package org.apache.apex.examples.twitter;
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/twitter/src/main/resources/META-INF/properties-TwitterKinesisDemo.xml
----------------------------------------------------------------------
diff --git a/examples/twitter/src/main/resources/META-INF/properties-TwitterKinesisDemo.xml b/examples/twitter/src/main/resources/META-INF/properties-TwitterKinesisDemo.xml
new file mode 100644
index 0000000..47b6531
--- /dev/null
+++ b/examples/twitter/src/main/resources/META-INF/properties-TwitterKinesisDemo.xml
@@ -0,0 +1,52 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+
+<!-- properties for the twitter kinesis example -->
+<configuration>
+
+ <property>
+ <name>dt.operator.FromKinesis.streamName</name>
+ <value>TwitterTag</value>
+ </property>
+ <property>
+ <name>dt.operator.FromKinesis.accessKey</name>
+ </property>
+ <property>
+ <name>dt.operator.FromKinesis.secretKey</name>
+ </property>
+ <property>
+ <name>dt.operator.FromKinesis.endPoint</name>
+ </property>
+ <property>
+ <name>dt.operator.ToKinesis.streamName</name>
+ <value>TwitterTag</value>
+ </property>
+ <property>
+ <name>dt.operator.ToKinesis.accessKey</name>
+ </property>
+ <property>
+ <name>dt.operator.ToKinesis.secretKey</name>
+ </property>
+ <property>
+ <name>dt.operator.ToKinesis.endPoint</name>
+ </property>
+
+</configuration>
[17/30] apex-malhar git commit: Renamed demos to examples. Packages
and artifactid names are changed as suggested.
Posted by th...@apache.org.
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/twitter/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/demos/twitter/src/main/resources/META-INF/properties.xml b/demos/twitter/src/main/resources/META-INF/properties.xml
deleted file mode 100644
index e3042fa..0000000
--- a/demos/twitter/src/main/resources/META-INF/properties.xml
+++ /dev/null
@@ -1,121 +0,0 @@
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-
-<!-- properties for rolling top words demo -->
-<configuration>
- <property>
- <name>dt.attr.MASTER_MEMORY_MB</name>
- <value>1256</value>
- </property>
- <!-- default operator size 256MB -->
- <property>
- <name>dt.application.*.operator.*.attr.MEMORY_MB</name>
- <value>256</value>
- </property>
- <property>
- <name>dt.application.*.operator.*.attr.JVM_OPTIONS</name>
- <value>-Xmx128M</value>
- </property>
-
- <!-- default buffer memory 256MB -->
- <property>
- <name>dt.application.*.operator.*.port.*.attr.BUFFER_MEMORY_MB</name>
- <value>256</value>
- </property>
- <property>
- <name>dt.operator.TweetSampler.consumerKey</name>
- </property>
- <property>
- <name>dt.operator.TweetSampler.consumerSecret</name>
- </property>
- <property>
- <name>dt.operator.TweetSampler.accessToken</name>
- </property>
- <property>
- <name>dt.operator.TweetSampler.accessTokenSecret</name>
- </property>
- <property>
- <name>dt.operator.TweetSampler.feedMultiplierVariance</name>
- <value>5</value>
- </property>
- <property>
- <name>dt.operator.TweetSampler.feedMultiplier</name>
- <value>20</value>
- </property>
-
- <!-- RollingTopWordsDemo -->
-
- <property>
- <name>dt.application.RollingTopWordsDemo.operator.TopCounter.topCount</name>
- <value>10</value>
- </property>
- <property>
- <name>dt.application.RollingTopWordsDemo.stream.TweetStream.locality</name>
- <value>CONTAINER_LOCAL</value>
- </property>
- <property>
- <name>dt.application.RollingTopWordsDemo.operator.SnapshotServer.embeddableQueryInfoProvider.topic</name>
- <value>TwitterWordsQuery</value>
- </property>
- <property>
- <name>dt.application.RollingTopWordsDemo.operator.QueryResult.topic</name>
- <value>TwitterWordsQueryResult</value>
- </property>
- <property>
- <name>dt.application.RollingTopWordsDemo.operator.QueryResult.numRetries</name>
- <value>2147483647</value>
- </property>
-
- <!-- TwitterDemo -->
-
- <property>
- <name>dt.application.TwitterDemo.operator.SnapshotServer.embeddableQueryInfoProvider.topic</name>
- <value>TwitterURLQuery</value>
- </property>
- <property>
- <name>dt.application.TwitterDemo.operator.QueryResult.topic</name>
- <value>TwitterURLQueryResult</value>
- </property>
- <property>
- <name>dt.application.TwitterDemo.operator.QueryResult.numRetries</name>
- <value>2147483647</value>
- </property>
- <property>
- <name>dt.application.TwitterDemo.operator.UniqueURLCounter.attr.APPLICATION_WINDOW_COUNT</name>
- <value>60</value>
- </property>
-
- <!-- TwitterTrendingDemo -->
-
- <property>
- <name>dt.application.TwitterTrendingDemo.operator.SnapshotServer.embeddableQueryInfoProvider.topic</name>
- <value>TwitterHashtagQueryDemo</value>
- </property>
- <property>
- <name>dt.application.TwitterTrendingDemo.operator.QueryResult.topic</name>
- <value>TwitterHashtagQueryResultDemo</value>
- </property>
- <property>
- <name>dt.application.TwitterTrendingDemo.operator.QueryResult.numRetries</name>
- <value>2147483647</value>
- </property>
-
-</configuration>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/twitter/src/main/resources/mysql.sql
----------------------------------------------------------------------
diff --git a/demos/twitter/src/main/resources/mysql.sql b/demos/twitter/src/main/resources/mysql.sql
deleted file mode 100644
index e0b97dd..0000000
--- a/demos/twitter/src/main/resources/mysql.sql
+++ /dev/null
@@ -1,35 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing,
--- software distributed under the License is distributed on an
--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
--- KIND, either express or implied. See the License for the
--- specific language governing permissions and limitations
--- under the License.
---
-
-DROP TABLE if exists tweets;
-CREATE TABLE tweets (
- window_id LONG NOT NULL,
- creation_date DATE,
- text VARCHAR(256) NOT NULL,
- userid VARCHAR(40) NOT NULL,
- KEY ( userid, creation_date)
- );
-
-drop table if exists dt_window_id_tracker;
-CREATE TABLE dt_window_id_tracker (
- dt_application_id VARCHAR(100) NOT NULL,
- dt_operator_id int(11) NOT NULL,
- dt_window_id bigint NOT NULL,
- UNIQUE (dt_application_id, dt_operator_id, dt_window_id)
-) ENGINE=MyISAM DEFAULT CHARSET=latin1;
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/twitter/src/main/resources/top_urls.tplg.properties
----------------------------------------------------------------------
diff --git a/demos/twitter/src/main/resources/top_urls.tplg.properties b/demos/twitter/src/main/resources/top_urls.tplg.properties
deleted file mode 100644
index c106d7d..0000000
--- a/demos/twitter/src/main/resources/top_urls.tplg.properties
+++ /dev/null
@@ -1,48 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-stram.node.twitterfeed.classname=com.datatorrent.example.twitter.TwitterSampleInput
-
-stram.stream.status.source=twitterfeed.output
-stram.stream.status.sinks=urlextractor.input
-
-stram.node.urlextractor.classname=com.datatorrent.example.twitter.TwitterStatusURLExtractor
-
-stram.stream.collapsedurls.source=urlextractor.output
-stram.stream.collapsedurls.sinks=
-
-stram.node.
-stram.stream.partitionedtf.input=twitterfeed.output
-stram.stream.partitionedtf.output=partitioned_counter.input
-stram.stream.partitionedtf.serdeClassname=com.datatorrent.example.twitter.URLSerDe
-
-stram.node.partitioned_counter.classname=com.datatorrent.example.twitter.PartitionedCounter
-stram.node.partitioned_counter.topCount=10
-
-stram.stream.merge_stream.input=partitioned_counter.output
-stram.stream.merge_stream.output=merge_counter.input
-stram.stream.merge_stream.serdeClassname=com.datatorrent.example.twitter.URLHolderSerde
-
-stram.node.merge_counter.classname=com.datatorrent.example.twitter.MergeSorter
-stram.node.merge_counter.topCount=10
-
-stram.stream.merged_stream.input=merge_counter.output
-stram.stream.merged_stream.output=console.input
-
-stram.node.console.classname=com.datatorrent.stream.ConsoleOutputStream
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/twitter/src/main/resources/twitterHashTagDataSchema.json
----------------------------------------------------------------------
diff --git a/demos/twitter/src/main/resources/twitterHashTagDataSchema.json b/demos/twitter/src/main/resources/twitterHashTagDataSchema.json
deleted file mode 100644
index 0c9296c..0000000
--- a/demos/twitter/src/main/resources/twitterHashTagDataSchema.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
- "values": [{"name": "hashtag", "type": "string"},
- {"name": "count", "type": "integer"}]
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/twitter/src/main/resources/twitterURLDataSchema.json
----------------------------------------------------------------------
diff --git a/demos/twitter/src/main/resources/twitterURLDataSchema.json b/demos/twitter/src/main/resources/twitterURLDataSchema.json
deleted file mode 100644
index ecf723e..0000000
--- a/demos/twitter/src/main/resources/twitterURLDataSchema.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
- "values": [{"name": "url", "type": "string"},
- {"name": "count", "type": "integer"}]
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/twitter/src/main/resources/twitterWordDataSchema.json
----------------------------------------------------------------------
diff --git a/demos/twitter/src/main/resources/twitterWordDataSchema.json b/demos/twitter/src/main/resources/twitterWordDataSchema.json
deleted file mode 100644
index 5e8e7c0..0000000
--- a/demos/twitter/src/main/resources/twitterWordDataSchema.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
- "values": [{"name": "word", "type": "string"},
- {"name": "count", "type": "integer"}]
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/twitter/src/test/java/com/datatorrent/demos/twitter/TwitterDumpApplicationTest.java
----------------------------------------------------------------------
diff --git a/demos/twitter/src/test/java/com/datatorrent/demos/twitter/TwitterDumpApplicationTest.java b/demos/twitter/src/test/java/com/datatorrent/demos/twitter/TwitterDumpApplicationTest.java
deleted file mode 100644
index cd211ff..0000000
--- a/demos/twitter/src/test/java/com/datatorrent/demos/twitter/TwitterDumpApplicationTest.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.twitter;
-
-import org.junit.Test;
-
-import org.apache.hadoop.conf.Configuration;
-
-import static org.junit.Assert.assertEquals;
-
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.LocalMode;
-
-/**
- * Test for the application which taps into the twitter's sample input stream and
- * dumps all the tweets into a database.
- */
-public class TwitterDumpApplicationTest
-{
- @Test
- public void testPopulateDAG() throws Exception
- {
- Configuration configuration = new Configuration(false);
-
- LocalMode lm = LocalMode.newInstance();
- DAG prepareDAG = lm.prepareDAG(new TwitterDumpApplication(), configuration);
- DAG clonedDAG = lm.cloneDAG();
-
- assertEquals("Serialization", prepareDAG, clonedDAG);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/twitter/src/test/java/com/datatorrent/demos/twitter/TwitterTopCounterTest.java
----------------------------------------------------------------------
diff --git a/demos/twitter/src/test/java/com/datatorrent/demos/twitter/TwitterTopCounterTest.java b/demos/twitter/src/test/java/com/datatorrent/demos/twitter/TwitterTopCounterTest.java
deleted file mode 100644
index 91a4e20..0000000
--- a/demos/twitter/src/test/java/com/datatorrent/demos/twitter/TwitterTopCounterTest.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.twitter;
-
-import org.junit.Test;
-import org.apache.hadoop.conf.Configuration;
-import com.datatorrent.api.LocalMode;
-import com.datatorrent.contrib.twitter.TwitterSampleInput;
-
-/**
- * Test the DAG declaration in local mode.
- */
-public class TwitterTopCounterTest
-{
- /**
- * This test requires twitter authentication setup and is skipped by default
- * (see {@link TwitterSampleInput}).
- *
- * @throws Exception
- */
- @Test
- public void testApplication() throws Exception
- {
- LocalMode lma = LocalMode.newInstance();
- new TwitterTopCounterApplication().populateDAG(lma.getDAG(), new Configuration(false));
- LocalMode.Controller lc = lma.getController();
- lc.run(120000);
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/twitter/src/test/java/com/datatorrent/demos/twitter/TwitterTopWordsTest.java
----------------------------------------------------------------------
diff --git a/demos/twitter/src/test/java/com/datatorrent/demos/twitter/TwitterTopWordsTest.java b/demos/twitter/src/test/java/com/datatorrent/demos/twitter/TwitterTopWordsTest.java
deleted file mode 100644
index 4ac2e8d..0000000
--- a/demos/twitter/src/test/java/com/datatorrent/demos/twitter/TwitterTopWordsTest.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.twitter;
-
-import org.junit.Test;
-import org.apache.hadoop.conf.Configuration;
-import com.datatorrent.api.LocalMode;
-import com.datatorrent.contrib.twitter.TwitterSampleInput;
-
-/**
- * Test the DAG declaration in local mode.
- */
-public class TwitterTopWordsTest
-{
- /**
- * This test requires twitter authentication setup and is skipped by default
- * (see {@link TwitterSampleInput}).
- *
- * @throws Exception
- */
- @Test
- public void testApplication() throws Exception
- {
- TwitterTopWordsApplication app = new TwitterTopWordsApplication();
- Configuration conf = new Configuration(false);
- conf.addResource("dt-site-rollingtopwords.xml");
- LocalMode lma = LocalMode.newInstance();
- lma.prepareDAG(app, conf);
- LocalMode.Controller lc = lma.getController();
- lc.run(120000);
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/twitter/src/test/resources/dt-site-rollingtopwords.xml
----------------------------------------------------------------------
diff --git a/demos/twitter/src/test/resources/dt-site-rollingtopwords.xml b/demos/twitter/src/test/resources/dt-site-rollingtopwords.xml
deleted file mode 100644
index b1d4153..0000000
--- a/demos/twitter/src/test/resources/dt-site-rollingtopwords.xml
+++ /dev/null
@@ -1,73 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<configuration>
-<property>
- <name>dt.attr.MASTER_MEMORY_MB</name>
- <value>1024</value>
- </property>
- <property>
- <name>dt.application.RollingTopWordsDemo.class</name>
- <value>com.datatorrent.demos.twitter.TwitterTopWordsApplication</value>
- <description>An alias for the application</description>
- </property>
- <property>
- <name>dt.application.RollingTopWordsDemo.operator.TopCounter.topCount
- </name>
- <value>10</value>
- </property>
- <property>
- <name>dt.application.RollingTopWordsDemo.stream.TweetStream.locality
- </name>
- <value>CONTAINER_LOCAL</value>
- </property>
- <property>
- <name>dt.application.RollingTopWordsDemo.stream.TwittedWords.locality
- </name>
- <value></value>
- </property>
- <property>
- <name>dt.application.RollingTopWordsDemo.stream.UniqueWordCounts.locality
- </name>
- <value></value>
- </property>
- <property>
- <name>dt.application.RollingTopWordsDemo.stream.TopWords.locality</name>
- <value></value>
- </property>
- <property>
- <name>dt.operator.TweetSampler.consumerKey</name>
- <value>r1DqM35iCTjbgLHf1R7rDbF5R</value>
- </property>
- <property>
- <name>dt.operator.TweetSampler.consumerSecret</name>
- <value>KBpZiR0glPzvZPm1Sa7sq9MCGQ2H2DrVChNtmYQcU75fwuHjed</value>
- </property>
- <property>
- <name>dt.operator.TweetSampler.accessToken</name>
- <value>2490355837-lXKev9vIGzftDjDu9LlyQiuGAqjYyPELFRyRpQo</value>
- </property>
- <property>
- <name>dt.operator.TweetSampler.accessTokenSecret</name>
- <value>lmAxZlFhcBqeTxmyatFT43fzshzrv6lsOAtHDsCBLjiuk</value>
- </property>
-
-</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/twitter/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/demos/twitter/src/test/resources/log4j.properties b/demos/twitter/src/test/resources/log4j.properties
deleted file mode 100644
index cf0d19e..0000000
--- a/demos/twitter/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,43 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-log4j.rootLogger=DEBUG,CONSOLE
-
-log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
-log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
-log4j.appender.CONSOLE.threshold=${test.log.console.threshold}
-test.log.console.threshold=DEBUG
-
-log4j.appender.RFA=org.apache.log4j.RollingFileAppender
-log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
-log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
-log4j.appender.RFA.File=/tmp/app.log
-
-# to enable, add SYSLOG to rootLogger
-log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender
-log4j.appender.SYSLOG.syslogHost=127.0.0.1
-log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout
-log4j.appender.SYSLOG.layout.conversionPattern=${dt.cid} %-5p [%t] %c{2} %x - %m%n
-log4j.appender.SYSLOG.Facility=LOCAL1
-
-log4j.logger.org=info
-#log4j.logger.org.apache.commons.beanutils=warn
-log4j.logger.com.datatorrent=debug
-log4j.logger.org.apache.apex=debug
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/uniquecount/pom.xml
----------------------------------------------------------------------
diff --git a/demos/uniquecount/pom.xml b/demos/uniquecount/pom.xml
deleted file mode 100644
index 7b402fc..0000000
--- a/demos/uniquecount/pom.xml
+++ /dev/null
@@ -1,50 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>uniquecount</artifactId>
- <packaging>jar</packaging>
-
- <name>Apache Apex Malhar Unique Count Demo</name>
- <description></description>
-
- <parent>
- <groupId>org.apache.apex</groupId>
- <artifactId>malhar-demos</artifactId>
- <version>3.7.0-SNAPSHOT</version>
- </parent>
-
- <properties>
- <skipTests>true</skipTests>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- <version>3.1</version>
- <scope>provided</scope>
- </dependency>
- </dependencies>
-
-</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/uniquecount/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/demos/uniquecount/src/assemble/appPackage.xml b/demos/uniquecount/src/assemble/appPackage.xml
deleted file mode 100644
index 4138cf2..0000000
--- a/demos/uniquecount/src/assemble/appPackage.xml
+++ /dev/null
@@ -1,59 +0,0 @@
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
- <id>appPackage</id>
- <formats>
- <format>jar</format>
- </formats>
- <includeBaseDirectory>false</includeBaseDirectory>
- <fileSets>
- <fileSet>
- <directory>${basedir}/target/</directory>
- <outputDirectory>/app</outputDirectory>
- <includes>
- <include>${project.artifactId}-${project.version}.jar</include>
- </includes>
- </fileSet>
- <fileSet>
- <directory>${basedir}/target/deps</directory>
- <outputDirectory>/lib</outputDirectory>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/site/conf</directory>
- <outputDirectory>/conf</outputDirectory>
- <includes>
- <include>*.xml</include>
- </includes>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/main/resources/META-INF</directory>
- <outputDirectory>/META-INF</outputDirectory>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/main/resources/app</directory>
- <outputDirectory>/app</outputDirectory>
- </fileSet>
- </fileSets>
-
-</assembly>
-
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/uniquecount/src/main/java/com/datatorrent/demos/uniquecount/Application.java
----------------------------------------------------------------------
diff --git a/demos/uniquecount/src/main/java/com/datatorrent/demos/uniquecount/Application.java b/demos/uniquecount/src/main/java/com/datatorrent/demos/uniquecount/Application.java
deleted file mode 100644
index 57ef1a1..0000000
--- a/demos/uniquecount/src/main/java/com/datatorrent/demos/uniquecount/Application.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.uniquecount;
-
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.Context;
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.DAG.Locality;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-
-import com.datatorrent.common.partitioner.StatelessPartitioner;
-import com.datatorrent.lib.algo.UniqueCounter;
-import com.datatorrent.lib.converter.MapToKeyHashValuePairConverter;
-import com.datatorrent.lib.io.ConsoleOutputOperator;
-import com.datatorrent.lib.stream.Counter;
-import com.datatorrent.lib.stream.StreamDuplicater;
-import com.datatorrent.lib.util.KeyHashValPair;
-
-/**
- * Application to demonstrate PartitionableUniqueCount operator. <br>
- * The input operator generate random keys, which is sent to
- * PartitionableUniqueCount operator initially partitioned into three partitions to
- * test unifier functionality, and output of the operator is sent to verifier to verify
- * that it generates correct result.
- *
- * @since 1.0.2
- */
-@ApplicationAnnotation(name = "UniqueValueCountDemo")
-public class Application implements StreamingApplication
-{
-
- @Override
- public void populateDAG(DAG dag, Configuration entries)
- {
- /* Generate random key-value pairs */
- RandomKeysGenerator randGen = dag.addOperator("randomgen", new RandomKeysGenerator());
-
-
- /* Initialize with three partition to start with */
- // UniqueCount1 uniqCount = dag.addOperator("uniqevalue", new UniqueCount1());
- UniqueCounter<Integer> uniqCount = dag.addOperator("uniqevalue", new UniqueCounter<Integer>());
-
- MapToKeyHashValuePairConverter<Integer, Integer> converter = dag.addOperator("converter", new MapToKeyHashValuePairConverter());
-
- uniqCount.setCumulative(false);
- dag.setAttribute(uniqCount, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<UniqueCounter<Integer>>(3));
-
- CountVerifier<Integer> verifier = dag.addOperator("verifier", new CountVerifier<Integer>());
- StreamDuplicater<KeyHashValPair<Integer, Integer>> dup = dag.addOperator("dup", new StreamDuplicater<KeyHashValPair<Integer, Integer>>());
- ConsoleOutputOperator output = dag.addOperator("output", new ConsoleOutputOperator());
-
- ConsoleOutputOperator successOutput = dag.addOperator("successoutput", new ConsoleOutputOperator());
- successOutput.setStringFormat("Success %d");
- ConsoleOutputOperator failureOutput = dag.addOperator("failureoutput", new ConsoleOutputOperator());
- failureOutput.setStringFormat("Failure %d");
-
- // success and failure counters.
- Counter successcounter = dag.addOperator("successcounter", new Counter());
- Counter failurecounter = dag.addOperator("failurecounter", new Counter());
-
- dag.addStream("datain", randGen.outPort, uniqCount.data);
- dag.addStream("dataverification0", randGen.verificationPort, verifier.in1);
- dag.addStream("convert", uniqCount.count, converter.input).setLocality(Locality.THREAD_LOCAL);
- dag.addStream("split", converter.output, dup.data);
- dag.addStream("consoutput", dup.out1, output.input);
- dag.addStream("dataverification1", dup.out2, verifier.in2);
- dag.addStream("successc", verifier.successPort, successcounter.input);
- dag.addStream("failurec", verifier.failurePort, failurecounter.input);
- dag.addStream("succconsoutput", successcounter.output, successOutput.input);
- dag.addStream("failconsoutput", failurecounter.output, failureOutput.input);
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/uniquecount/src/main/java/com/datatorrent/demos/uniquecount/CountVerifier.java
----------------------------------------------------------------------
diff --git a/demos/uniquecount/src/main/java/com/datatorrent/demos/uniquecount/CountVerifier.java b/demos/uniquecount/src/main/java/com/datatorrent/demos/uniquecount/CountVerifier.java
deleted file mode 100644
index d201037..0000000
--- a/demos/uniquecount/src/main/java/com/datatorrent/demos/uniquecount/CountVerifier.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.uniquecount;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import com.datatorrent.api.Context;
-import com.datatorrent.api.DefaultInputPort;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.api.Operator;
-import com.datatorrent.api.annotation.OutputPortFieldAnnotation;
-import com.datatorrent.lib.util.KeyHashValPair;
-
-/*
-Compare results and print non-matching values to console.
- */
-/**
- * <p>CountVerifier class.</p>
- *
- * @since 1.0.2
- */
-public class CountVerifier<K> implements Operator
-{
- HashMap<K, Integer> map1 = new HashMap<K, Integer>();
- HashMap<K, Integer> map2 = new HashMap<K, Integer>();
-
- public final transient DefaultInputPort<KeyHashValPair<K, Integer>> in1 = new DefaultInputPort<KeyHashValPair<K, Integer>>()
- {
- @Override
- public void process(KeyHashValPair<K, Integer> tuple)
- {
- processTuple(tuple, map1);
- }
- };
-
- public final transient DefaultInputPort<KeyHashValPair<K, Integer>> in2 = new DefaultInputPort<KeyHashValPair<K, Integer>>()
- {
- @Override
- public void process(KeyHashValPair<K, Integer> tuple)
- {
- processTuple(tuple, map2);
- }
- };
-
- void processTuple(KeyHashValPair<K, Integer> tuple, HashMap<K, Integer> map)
- {
- map.put(tuple.getKey(), tuple.getValue());
- }
-
- @OutputPortFieldAnnotation(optional = true)
- public final transient DefaultOutputPort<Integer> successPort = new DefaultOutputPort<Integer>();
- @OutputPortFieldAnnotation(optional = true)
- public final transient DefaultOutputPort<Integer> failurePort = new DefaultOutputPort<Integer>();
-
- @Override
- public void beginWindow(long l)
- {
-
- }
-
- @Override
- public void endWindow()
- {
- int failureCount = 0;
- for (Map.Entry<K, Integer> e : map1.entrySet()) {
- K key = e.getKey();
- int val = map2.get(key);
- if (val != e.getValue()) {
- failureCount++;
- }
- }
- if (failureCount != 0) {
- failurePort.emit(failureCount);
- } else {
- successPort.emit(map1.size());
- }
- }
-
- @Override
- public void setup(Context.OperatorContext operatorContext)
- {
-
- }
-
- @Override
- public void teardown()
- {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/uniquecount/src/main/java/com/datatorrent/demos/uniquecount/RandomDataGenerator.java
----------------------------------------------------------------------
diff --git a/demos/uniquecount/src/main/java/com/datatorrent/demos/uniquecount/RandomDataGenerator.java b/demos/uniquecount/src/main/java/com/datatorrent/demos/uniquecount/RandomDataGenerator.java
deleted file mode 100644
index e806759..0000000
--- a/demos/uniquecount/src/main/java/com/datatorrent/demos/uniquecount/RandomDataGenerator.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.uniquecount;
-
-import java.util.HashMap;
-import java.util.Random;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.datatorrent.api.Context;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.api.InputOperator;
-import com.datatorrent.lib.util.KeyValPair;
-
-/**
- * Generate random Key value pairs.
- * key is string and value is int, it emits the pair as KeyValPair on outPort,
- *
- * @since 1.0.2
- */
-public class RandomDataGenerator implements InputOperator
-{
- public final transient DefaultOutputPort<KeyValPair<String, Object>> outPort = new DefaultOutputPort<KeyValPair<String, Object>>();
- private HashMap<String, Integer> dataInfo;
- private final transient Logger LOG = LoggerFactory.getLogger(RandomDataGenerator.class);
- private int count;
- private int sleepMs = 10;
- private int keyRange = 100;
- private int valueRange = 100;
- private long tupleBlast = 10000;
- private Random random;
-
- public RandomDataGenerator()
- {
- random = new Random();
- }
-
- @Override
- public void emitTuples()
- {
- for (int i = 0; i < tupleBlast; i++) {
- String key = String.valueOf(random.nextInt(keyRange));
- int val = random.nextInt(valueRange);
- outPort.emit(new KeyValPair<String, Object>(key, val));
- }
- try {
- Thread.sleep(sleepMs);
- } catch (Exception ex) {
- LOG.error(ex.getMessage());
- }
- count++;
- }
-
- public int getSleepMs()
- {
- return sleepMs;
- }
-
- public void setSleepMs(int sleepMs)
- {
- this.sleepMs = sleepMs;
- }
-
- public long getTupleBlast()
- {
- return tupleBlast;
- }
-
- public void setTupleBlast(long tupleBlast)
- {
- this.tupleBlast = tupleBlast;
- }
-
- @Override
- public void beginWindow(long l)
- {
-
- }
-
- @Override
- public void endWindow()
- {
- LOG.debug("emitTuples called " + count + " times in this window");
- count = 0;
- }
-
- @Override
- public void setup(Context.OperatorContext operatorContext)
- {
-
- }
-
- @Override
- public void teardown()
- {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/uniquecount/src/main/java/com/datatorrent/demos/uniquecount/RandomKeyValues.java
----------------------------------------------------------------------
diff --git a/demos/uniquecount/src/main/java/com/datatorrent/demos/uniquecount/RandomKeyValues.java b/demos/uniquecount/src/main/java/com/datatorrent/demos/uniquecount/RandomKeyValues.java
deleted file mode 100644
index 28f3bc0..0000000
--- a/demos/uniquecount/src/main/java/com/datatorrent/demos/uniquecount/RandomKeyValues.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.uniquecount;
-
-import java.util.BitSet;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Random;
-
-import com.datatorrent.api.Context.OperatorContext;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.api.InputOperator;
-import com.datatorrent.lib.util.KeyValPair;
-
-/**
- * Input port operator for generating random values on keys. <br>
- * Key(s) : key + integer in range between 0 and numKeys <br>
- * Value(s) : integer in range of 0 to numValuesPerKeys <br>
- *
- * @since 0.9.3
- */
-public class RandomKeyValues implements InputOperator
-{
- public final transient DefaultOutputPort<KeyValPair<String, Object>> outport = new DefaultOutputPort<KeyValPair<String, Object>>();
- private Random random = new Random(11111);
- private int numKeys;
- private int numValuesPerKeys;
- private int tuppleBlast = 1000;
- private int emitDelay = 20; /* 20 ms */
-
- /* For verification */
- private Map<Integer, BitSet> history = new HashMap<Integer, BitSet>();
-
- public RandomKeyValues()
- {
- this.numKeys = 100;
- this.numValuesPerKeys = 100;
- }
-
- public RandomKeyValues(int keys, int values)
- {
- this.numKeys = keys;
- this.numValuesPerKeys = values;
- }
-
- @Override
- public void beginWindow(long windowId)
- {
- }
-
- @Override
- public void endWindow()
- {
- }
-
- @Override
- public void setup(OperatorContext context)
- {
- }
-
- @Override
- public void teardown()
- {
- }
-
- @Override
- public void emitTuples()
- {
- /* generate tuples randomly, */
- for (int i = 0; i < tuppleBlast; i++) {
- int intKey = random.nextInt(numKeys);
- String key = "key" + String.valueOf(intKey);
- int value = random.nextInt(numValuesPerKeys);
-
- // update history for verifying later.
- BitSet bmap = history.get(intKey);
- if (bmap == null) {
- bmap = new BitSet();
- history.put(intKey, bmap);
- }
- bmap.set(value);
-
- // emit the key with value.
- outport.emit(new KeyValPair<String, Object>(key, value));
- }
- try {
- Thread.sleep(emitDelay);
- } catch (Exception e) {
- // Ignore.
- }
- }
-
- public int getNumKeys()
- {
- return numKeys;
- }
-
- public void setNumKeys(int numKeys)
- {
- this.numKeys = numKeys;
- }
-
- public int getNumValuesPerKeys()
- {
- return numValuesPerKeys;
- }
-
- public void setNumValuesPerKeys(int numValuesPerKeys)
- {
- this.numValuesPerKeys = numValuesPerKeys;
- }
-
- public int getTuppleBlast()
- {
- return tuppleBlast;
- }
-
- public void setTuppleBlast(int tuppleBlast)
- {
- this.tuppleBlast = tuppleBlast;
- }
-
- public int getEmitDelay()
- {
- return emitDelay;
- }
-
- public void setEmitDelay(int emitDelay)
- {
- this.emitDelay = emitDelay;
- }
-
- public void debug()
- {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/uniquecount/src/main/java/com/datatorrent/demos/uniquecount/RandomKeysGenerator.java
----------------------------------------------------------------------
diff --git a/demos/uniquecount/src/main/java/com/datatorrent/demos/uniquecount/RandomKeysGenerator.java b/demos/uniquecount/src/main/java/com/datatorrent/demos/uniquecount/RandomKeysGenerator.java
deleted file mode 100644
index eb9d22c..0000000
--- a/demos/uniquecount/src/main/java/com/datatorrent/demos/uniquecount/RandomKeysGenerator.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.uniquecount;
-
-import java.util.Date;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Random;
-
-import org.apache.commons.lang3.mutable.MutableInt;
-
-import com.datatorrent.api.Context;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.api.InputOperator;
-import com.datatorrent.api.annotation.OutputPortFieldAnnotation;
-import com.datatorrent.lib.util.KeyHashValPair;
-
-/*
- Generate random keys.
- */
-/**
- * <p>RandomKeysGenerator class.</p>
- *
- * @since 1.0.2
- */
-public class RandomKeysGenerator implements InputOperator
-{
-
- protected int numKeys = 100;
- protected int tupleBlast = 15000;
- protected long sleepTime = 0;
- protected Map<Integer, MutableInt> history = new HashMap<Integer, MutableInt>();
- private Random random = new Random();
- private Date date = new Date();
- private long start;
-
- @OutputPortFieldAnnotation(optional = false)
- public transient DefaultOutputPort<Integer> outPort = new DefaultOutputPort<Integer>();
-
- @OutputPortFieldAnnotation(optional = true)
- public transient DefaultOutputPort<KeyHashValPair<Integer, Integer>> verificationPort =
- new DefaultOutputPort<KeyHashValPair<Integer, Integer>>();
-
- @Override
- public void emitTuples()
- {
- for (int i = 0; i < tupleBlast; i++) {
- int key = random.nextInt(numKeys);
- outPort.emit(key);
-
-
- if (verificationPort.isConnected()) {
- // maintain history for later verification.
- MutableInt count = history.get(key);
- if (count == null) {
- count = new MutableInt(0);
- history.put(key, count);
- }
- count.increment();
- }
-
- }
- try {
- if (sleepTime != 0) {
- Thread.sleep(sleepTime);
- }
- } catch (Exception ex) {
- // Ignore.
- }
- }
-
- public RandomKeysGenerator()
- {
- start = date.getTime();
- }
-
- @Override
- public void beginWindow(long l)
- {
-
- }
-
- @Override
- public void endWindow()
- {
-
- if (verificationPort.isConnected()) {
- for (Map.Entry<Integer, MutableInt> e : history.entrySet()) {
- verificationPort.emit(new KeyHashValPair<Integer, Integer>(e.getKey(), e.getValue().toInteger()));
- }
- history.clear();
- }
-
- }
-
- @Override
- public void setup(Context.OperatorContext operatorContext)
- {
-
- }
-
- @Override
- public void teardown()
- {
-
- }
-
- public int getNumKeys()
- {
- return numKeys;
- }
-
- public void setNumKeys(int numKeys)
- {
- this.numKeys = numKeys;
- }
-
- public int getTupleBlast()
- {
- return tupleBlast;
- }
-
- public void setTupleBlast(int tupleBlast)
- {
- this.tupleBlast = tupleBlast;
- }
-
- public long getSleepTime()
- {
- return sleepTime;
- }
-
- public void setSleepTime(long sleepTime)
- {
- this.sleepTime = sleepTime;
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/uniquecount/src/main/java/com/datatorrent/demos/uniquecount/UniqueKeyValCountDemo.java
----------------------------------------------------------------------
diff --git a/demos/uniquecount/src/main/java/com/datatorrent/demos/uniquecount/UniqueKeyValCountDemo.java b/demos/uniquecount/src/main/java/com/datatorrent/demos/uniquecount/UniqueKeyValCountDemo.java
deleted file mode 100644
index eb9e392..0000000
--- a/demos/uniquecount/src/main/java/com/datatorrent/demos/uniquecount/UniqueKeyValCountDemo.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.uniquecount;
-
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.Context;
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.DAG.Locality;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-
-import com.datatorrent.common.partitioner.StatelessPartitioner;
-
-import com.datatorrent.lib.algo.UniqueCounter;
-import com.datatorrent.lib.converter.MapToKeyHashValuePairConverter;
-import com.datatorrent.lib.io.ConsoleOutputOperator;
-import com.datatorrent.lib.util.KeyValPair;
-
-/**
- * <p>UniqueKeyValCountDemo class.</p>
- *
- * @since 1.0.2
- */
-@ApplicationAnnotation(name = "UniqueKeyValueCountDemo")
-public class UniqueKeyValCountDemo implements StreamingApplication
-{
-
- @Override
- public void populateDAG(DAG dag, Configuration entries)
- {
- /* Generate random key-value pairs */
- RandomDataGenerator randGen = dag.addOperator("randomgen", new RandomDataGenerator());
-
- /* Initialize with three partition to start with */
- UniqueCounter<KeyValPair<String, Object>> uniqCount =
- dag.addOperator("uniqevalue", new UniqueCounter<KeyValPair<String, Object>>());
- MapToKeyHashValuePairConverter<KeyValPair<String, Object>, Integer> converter = dag.addOperator("converter", new MapToKeyHashValuePairConverter());
- uniqCount.setCumulative(false);
- dag.setAttribute(randGen, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<UniqueCounter<KeyValPair<String, Object>>>(3));
-
- ConsoleOutputOperator output = dag.addOperator("output", new ConsoleOutputOperator());
-
- dag.addStream("datain", randGen.outPort, uniqCount.data);
- dag.addStream("convert", uniqCount.count, converter.input).setLocality(Locality.THREAD_LOCAL);
- dag.addStream("consoutput", converter.output, output.input);
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/uniquecount/src/main/java/com/datatorrent/demos/uniquecount/package-info.java
----------------------------------------------------------------------
diff --git a/demos/uniquecount/src/main/java/com/datatorrent/demos/uniquecount/package-info.java b/demos/uniquecount/src/main/java/com/datatorrent/demos/uniquecount/package-info.java
deleted file mode 100644
index 6f81c0d..0000000
--- a/demos/uniquecount/src/main/java/com/datatorrent/demos/uniquecount/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- Demo Application for new Paritionable UniqueCount Operator.
- */
-package com.datatorrent.demos.uniquecount;
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/uniquecount/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/demos/uniquecount/src/main/resources/META-INF/properties.xml b/demos/uniquecount/src/main/resources/META-INF/properties.xml
deleted file mode 100644
index 8742328..0000000
--- a/demos/uniquecount/src/main/resources/META-INF/properties.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-<?xml version="1.0"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<configuration>
- <!--
- <property>
- <name>dt.application.{appName}.operator.{opName}.prop.{propName}</name>
- <value>some-default-value (if value is not specified, it is required from the user or custom config when launching)</value>
- </property>
- -->
-</configuration>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/uniquecount/src/site/conf/my-app-conf1.xml
----------------------------------------------------------------------
diff --git a/demos/uniquecount/src/site/conf/my-app-conf1.xml b/demos/uniquecount/src/site/conf/my-app-conf1.xml
deleted file mode 100644
index f35873b..0000000
--- a/demos/uniquecount/src/site/conf/my-app-conf1.xml
+++ /dev/null
@@ -1,27 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<configuration>
- <property>
- <name>dt.attr.MASTER_MEMORY_MB</name>
- <value>1024</value>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/uniquecount/src/test/java/com/datatorrent/demos/uniquecount/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/demos/uniquecount/src/test/java/com/datatorrent/demos/uniquecount/ApplicationTest.java b/demos/uniquecount/src/test/java/com/datatorrent/demos/uniquecount/ApplicationTest.java
deleted file mode 100644
index 66a0af1..0000000
--- a/demos/uniquecount/src/test/java/com/datatorrent/demos/uniquecount/ApplicationTest.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.uniquecount;
-
-import org.junit.Test;
-import org.apache.hadoop.conf.Configuration;
-import com.datatorrent.api.LocalMode;
-
-/**
- * Test the DAG declaration in local mode.
- */
-public class ApplicationTest
-{
- @Test
- public void testApplication() throws Exception
- {
- LocalMode lma = LocalMode.newInstance();
- new Application().populateDAG(lma.getDAG(), new Configuration(false));
- LocalMode.Controller lc = lma.getController();
- lc.run(10000);
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/uniquecount/src/test/java/com/datatorrent/demos/uniquecount/UniqueKeyValDemoTest.java
----------------------------------------------------------------------
diff --git a/demos/uniquecount/src/test/java/com/datatorrent/demos/uniquecount/UniqueKeyValDemoTest.java b/demos/uniquecount/src/test/java/com/datatorrent/demos/uniquecount/UniqueKeyValDemoTest.java
deleted file mode 100644
index a198247..0000000
--- a/demos/uniquecount/src/test/java/com/datatorrent/demos/uniquecount/UniqueKeyValDemoTest.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.uniquecount;
-
-import org.junit.Test;
-import org.apache.hadoop.conf.Configuration;
-import com.datatorrent.api.LocalMode;
-
-/**
- * Test the DAG declaration in local mode.
- */
-public class UniqueKeyValDemoTest
-{
- @Test
- public void testApplication() throws Exception
- {
- LocalMode lma = LocalMode.newInstance();
- new UniqueKeyValCountDemo().populateDAG(lma.getDAG(), new Configuration(false));
- LocalMode.Controller lc = lma.getController();
- lc.run(10000);
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/uniquecount/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/demos/uniquecount/src/test/resources/log4j.properties b/demos/uniquecount/src/test/resources/log4j.properties
deleted file mode 100644
index cf0d19e..0000000
--- a/demos/uniquecount/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,43 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-log4j.rootLogger=DEBUG,CONSOLE
-
-log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
-log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
-log4j.appender.CONSOLE.threshold=${test.log.console.threshold}
-test.log.console.threshold=DEBUG
-
-log4j.appender.RFA=org.apache.log4j.RollingFileAppender
-log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
-log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
-log4j.appender.RFA.File=/tmp/app.log
-
-# to enable, add SYSLOG to rootLogger
-log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender
-log4j.appender.SYSLOG.syslogHost=127.0.0.1
-log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout
-log4j.appender.SYSLOG.layout.conversionPattern=${dt.cid} %-5p [%t] %c{2} %x - %m%n
-log4j.appender.SYSLOG.Facility=LOCAL1
-
-log4j.logger.org=info
-#log4j.logger.org.apache.commons.beanutils=warn
-log4j.logger.com.datatorrent=debug
-log4j.logger.org.apache.apex=debug
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/wordcount/pom.xml
----------------------------------------------------------------------
diff --git a/demos/wordcount/pom.xml b/demos/wordcount/pom.xml
deleted file mode 100644
index 410daea..0000000
--- a/demos/wordcount/pom.xml
+++ /dev/null
@@ -1,49 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>wordcount-demo</artifactId>
- <packaging>jar</packaging>
-
- <name>Apache Apex Malhar Wordcount Demo</name>
- <description>A very simple application that demonstrates Apex Platform\u2019s streaming window feature.</description>
-
- <parent>
- <groupId>org.apache.apex</groupId>
- <artifactId>malhar-demos</artifactId>
- <version>3.7.0-SNAPSHOT</version>
- </parent>
-
- <properties>
- <skipTests>true</skipTests>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>it.unimi.dsi</groupId>
- <artifactId>fastutil</artifactId>
- <version>6.6.4</version>
- </dependency>
- </dependencies>
-
-</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/wordcount/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/demos/wordcount/src/assemble/appPackage.xml b/demos/wordcount/src/assemble/appPackage.xml
deleted file mode 100644
index 4138cf2..0000000
--- a/demos/wordcount/src/assemble/appPackage.xml
+++ /dev/null
@@ -1,59 +0,0 @@
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
- <id>appPackage</id>
- <formats>
- <format>jar</format>
- </formats>
- <includeBaseDirectory>false</includeBaseDirectory>
- <fileSets>
- <fileSet>
- <directory>${basedir}/target/</directory>
- <outputDirectory>/app</outputDirectory>
- <includes>
- <include>${project.artifactId}-${project.version}.jar</include>
- </includes>
- </fileSet>
- <fileSet>
- <directory>${basedir}/target/deps</directory>
- <outputDirectory>/lib</outputDirectory>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/site/conf</directory>
- <outputDirectory>/conf</outputDirectory>
- <includes>
- <include>*.xml</include>
- </includes>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/main/resources/META-INF</directory>
- <outputDirectory>/META-INF</outputDirectory>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/main/resources/app</directory>
- <outputDirectory>/app</outputDirectory>
- </fileSet>
- </fileSets>
-
-</assembly>
-
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/wordcount/src/main/java/com/datatorrent/demos/wordcount/Application.java
----------------------------------------------------------------------
diff --git a/demos/wordcount/src/main/java/com/datatorrent/demos/wordcount/Application.java b/demos/wordcount/src/main/java/com/datatorrent/demos/wordcount/Application.java
deleted file mode 100644
index d0512cf..0000000
--- a/demos/wordcount/src/main/java/com/datatorrent/demos/wordcount/Application.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.wordcount;
-
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-import com.datatorrent.lib.algo.UniqueCounter;
-import com.datatorrent.lib.io.ConsoleOutputOperator;
-
-/**
- * Simple Word Count Demo : <br>
- * This is application to count total occurrence of each word from file or any
- * stream. <br>
- * <br>
- *
- * Functional Description : <br>
- * This demo declares custom input operator to read data file set by user. <br>
- * This input operator can be replaced by any stream input operator. <br>
- * <br>
- *
- * Custom Attribute(s) : None <br>
- * <br>
- *
- * Input Adapter : <br>
- * Word input operator opens user specified data file and streams each line to
- * application. <br>
- * <br>
- *
- * Output Adapter : <br>
- * Output values are written to console through ConsoleOutputOerator<br>
- * If needed you can use other output adapters<br>
- * <br>
- * <p>
- * Running Java Test or Main app in IDE:
- *
- * <pre>
- * LocalMode.runApp(new Application(), 600000); // 10 min run
- * </pre>
- *
- * Run Success : <br>
- * For successful deployment and run, user should see following output on console: <br>
- * </pre>
- * {developed=1} {bush\u2019s=2} {roster=1} {council=1} {mankiw=1} {academia=1}
- * {of=6} {help=1} {are=1} {presidential=1}
- * </pre> <br> <br>
- *
- * Scaling Options : <br>
- * This operator app can not be scaled, please look at implementation {@link com.datatorrent.lib.algo.UniqueCounterEach} <br> <br>
- *
- * Application DAG : <br>
- * <img src="doc-files/UniqueWordCounter.jpg" width=600px > <br>
- *
- * Streaming Window Size : 500ms
- * Operator Details : <br>
- * <ul>
- * <li>
- * <p><b> The operator wordinput : </b> This operator opens local file, reads each line and sends each word to application.
- * This can replaced by any input stream by user. <br>
- * Class : {@link com.datatorrent.demos.wordcount.WordCountInputOperator} <br>
- * Operator Application Window Count : 1 <br>
- * StateFull : No
- * </li>
- * <li>
- * <p><b> The operator count : </b> This operator aggregates unique key count over one window count(app). <br>
- * Class : {@link com.datatorrent.lib.algo.UniqueCounterEach} <br>
- * Operator Application Window Count : 1 <br>
- * StateFull : No
- * </li>
- * <li>
- * <p><b>The operator Console: </b> This operator just outputs the input tuples to the console (or stdout).
- * This case it emits unique count of each word over 500ms.
- * </li>
- * </ul>
- *
- * @since 0.3.2
- */
-@ApplicationAnnotation(name = "WordCountDemo")
-public class Application implements StreamingApplication
-{
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- WordCountInputOperator input = dag.addOperator("wordinput", new WordCountInputOperator());
- UniqueCounter<String> wordCount = dag.addOperator("count", new UniqueCounter<String>());
- dag.addStream("wordinput-count", input.outputPort, wordCount.data);
- ConsoleOutputOperator consoleOperator = dag.addOperator("console", new ConsoleOutputOperator());
- dag.addStream("count-console",wordCount.count, consoleOperator.input);
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/wordcount/src/main/java/com/datatorrent/demos/wordcount/ApplicationWithQuerySupport.java
----------------------------------------------------------------------
diff --git a/demos/wordcount/src/main/java/com/datatorrent/demos/wordcount/ApplicationWithQuerySupport.java b/demos/wordcount/src/main/java/com/datatorrent/demos/wordcount/ApplicationWithQuerySupport.java
deleted file mode 100644
index 7e5bb93..0000000
--- a/demos/wordcount/src/main/java/com/datatorrent/demos/wordcount/ApplicationWithQuerySupport.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.wordcount;
-
-import java.net.URI;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.Operator;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-
-import com.datatorrent.lib.appdata.schemas.SchemaUtils;
-import com.datatorrent.lib.appdata.snapshot.AppDataSnapshotServerMap;
-import com.datatorrent.lib.io.ConsoleOutputOperator;
-import com.datatorrent.lib.io.PubSubWebSocketAppDataQuery;
-import com.datatorrent.lib.io.PubSubWebSocketAppDataResult;
-
-/**
- * Simple demo that computes word frequencies from any file dropped into a
- * monitored directory. It outputs the top N word-frequency pairs for each file
- * as well globally across all files.
- * <p>
- * Each input file generates a corresponding output file in the output directory
- * containing the top N pairs for that file. The output is also written
- * to an internal store to support visualization in the UI via queries.
- * <p>
- * @since 3.2.0
- */
-@ApplicationAnnotation(name = "TopNWordsWithQueries")
-public class ApplicationWithQuerySupport implements StreamingApplication
-{
- private static final Logger LOG = LoggerFactory.getLogger(ApplicationWithQuerySupport.class);
-
- /**
- * Name of schema file.
- */
- public static final String SNAPSHOT_SCHEMA = "WordDataSchema.json";
-
- /**
- * Populates the DAG with operators and connecting streams
- *
- * @param dag The directed acyclic graph of operators to populate
- * @param conf The configuration
- */
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- // create operators
- LineReader lineReader = dag.addOperator("lineReader", new LineReader());
- WordReader wordReader = dag.addOperator("wordReader", new WordReader());
- WindowWordCount windowWordCount = dag.addOperator("windowWordCount", new WindowWordCount());
- FileWordCount fileWordCount = dag.addOperator("fileWordCount", new FileWordCount());
- WordCountWriter wcWriter = dag.addOperator("wcWriter", new WordCountWriter());
- ConsoleOutputOperator console = dag.addOperator("console", new ConsoleOutputOperator());
- console.setStringFormat("wordCount: %s");
-
- // create streams
-
- dag.addStream("lines", lineReader.output, wordReader.input);
- dag.addStream("control", lineReader.control, fileWordCount.control);
- dag.addStream("words", wordReader.output, windowWordCount.input);
- dag.addStream("windowWordCounts", windowWordCount.output, fileWordCount.input);
- dag.addStream("fileWordCounts", fileWordCount.fileOutput, wcWriter.input);
-
- String gatewayAddress = dag.getValue(DAG.GATEWAY_CONNECT_ADDRESS);
-
- if (!StringUtils.isEmpty(gatewayAddress)) { // add query support
- URI uri = URI.create("ws://" + gatewayAddress + "/pubsub");
-
- AppDataSnapshotServerMap snapshotServerFile
- = dag.addOperator("snapshotServerFile", new AppDataSnapshotServerMap());
- AppDataSnapshotServerMap snapshotServerGlobal
- = dag.addOperator("snapshotServerGlobal", new AppDataSnapshotServerMap());
-
- String snapshotServerJSON = SchemaUtils.jarResourceFileToString(SNAPSHOT_SCHEMA);
- snapshotServerFile.setSnapshotSchemaJSON(snapshotServerJSON);
- snapshotServerGlobal.setSnapshotSchemaJSON(snapshotServerJSON);
-
- PubSubWebSocketAppDataQuery wsQueryFile = new PubSubWebSocketAppDataQuery();
- PubSubWebSocketAppDataQuery wsQueryGlobal = new PubSubWebSocketAppDataQuery();
- wsQueryFile.setUri(uri);
- wsQueryGlobal.setUri(uri);
-
- snapshotServerFile.setEmbeddableQueryInfoProvider(wsQueryFile);
- snapshotServerGlobal.setEmbeddableQueryInfoProvider(wsQueryGlobal);
-
- PubSubWebSocketAppDataResult wsResultFile
- = dag.addOperator("wsResultFile", new PubSubWebSocketAppDataResult());
- PubSubWebSocketAppDataResult wsResultGlobal
- = dag.addOperator("wsResultGlobal", new PubSubWebSocketAppDataResult());
- wsResultFile.setUri(uri);
- wsResultGlobal.setUri(uri);
-
- Operator.InputPort<String> queryResultFilePort = wsResultFile.input;
- Operator.InputPort<String> queryResultGlobalPort = wsResultGlobal.input;
-
- dag.addStream("WordCountsFile", fileWordCount.outputPerFile, snapshotServerFile.input, console.input);
- dag.addStream("WordCountsGlobal", fileWordCount.outputGlobal, snapshotServerGlobal.input);
-
- dag.addStream("ResultFile", snapshotServerFile.queryResult, queryResultFilePort);
- dag.addStream("ResultGlobal", snapshotServerGlobal.queryResult, queryResultGlobalPort);
- } else {
- //throw new RuntimeException("Error: No GATEWAY_CONNECT_ADDRESS");
- dag.addStream("WordCounts", fileWordCount.outputPerFile, console.input);
- }
-
- LOG.info("done with populateDAG, isDebugEnabled = " + LOG.isDebugEnabled());
- LOG.info("Returning from populateDAG");
- }
-
-}