You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@apex.apache.org by th...@apache.org on 2017/03/07 06:58:35 UTC
[30/30] apex-malhar git commit: Renamed demos to examples. Packages
and artifactid names are changed as suggested.
Renamed demos to examples.
Packages and artifactid names are changed as suggested.
Project: http://git-wip-us.apache.org/repos/asf/apex-malhar/repo
Commit: http://git-wip-us.apache.org/repos/asf/apex-malhar/commit/d5bf96ca
Tree: http://git-wip-us.apache.org/repos/asf/apex-malhar/tree/d5bf96ca
Diff: http://git-wip-us.apache.org/repos/asf/apex-malhar/diff/d5bf96ca
Branch: refs/heads/master
Commit: d5bf96cac175cfd7e2d4a0effad1f057c0e8c604
Parents: 4b36bf3
Author: Apex Dev <de...@apex.apache.org>
Authored: Fri Feb 24 16:27:28 2017 -0800
Committer: Lakshmi Prasanna Velineni <la...@datatorrent.com>
Committed: Mon Mar 6 22:13:24 2017 -0800
----------------------------------------------------------------------
README.md | 4 +-
demos/distributedistinct/pom.xml | 41 -
.../src/assemble/appPackage.xml | 59 -
.../demos/distributeddistinct/Application.java | 69 -
.../distributeddistinct/CountVerifier.java | 103 -
.../IntegerUniqueValueCountAppender.java | 95 -
.../RandomKeyValGenerator.java | 186 -
.../StatefulApplication.java | 76 -
.../UniqueValueCountAppender.java | 236 -
.../src/main/resources/META-INF/properties.xml | 29 -
.../src/site/conf/my-app-conf1.xml | 27 -
.../distributeddistinct/ApplicationTest.java | 32 -
.../DistributedDistinctTest.java | 200 -
.../StatefulApplicationTest.java | 78 -
.../StatefulUniqueCountTest.java | 248 -
.../src/test/resources/log4j.properties | 43 -
demos/echoserver/pom.xml | 38 -
demos/echoserver/src/assemble/appPackage.xml | 59 -
.../demos/echoserver/Application.java | 43 -
.../datatorrent/demos/echoserver/Message.java | 30 -
.../demos/echoserver/MessageReceiver.java | 156 -
.../demos/echoserver/MessageResponder.java | 81 -
.../demos/echoserver/NetworkManager.java | 249 -
.../src/main/resources/META-INF/properties.xml | 38 -
demos/echoserver/src/site/conf/my-app-conf1.xml | 27 -
.../demos/echoserver/ApplicationTest.java | 53 -
.../src/test/resources/log4j.properties | 43 -
demos/frauddetect/pom.xml | 49 -
demos/frauddetect/src/assemble/appPackage.xml | 59 -
.../demos/frauddetect/Application.java | 151 -
.../demos/frauddetect/AverageAlertData.java | 37 -
.../frauddetect/AverageAlertingOperator.java | 185 -
.../frauddetect/BankIdNumberAlertData.java | 36 -
.../demos/frauddetect/BankIdNumberKey.java | 65 -
.../BankIdNumberSamplerOperator.java | 216 -
.../demos/frauddetect/CreditCardAlertData.java | 38 -
.../CreditCardAmountSamplerOperator.java | 206 -
.../demos/frauddetect/CreditCardData.java | 35 -
.../demos/frauddetect/MerchantKey.java | 128 -
.../frauddetect/MerchantQueryInputHandler.java | 69 -
.../demos/frauddetect/MerchantTransaction.java | 202 -
.../MerchantTransactionBucketOperator.java | 148 -
.../MerchantTransactionGenerator.java | 210 -
.../MerchantTransactionInputHandler.java | 116 -
.../frauddetect/SlidingWindowSumKeyVal.java | 99 -
.../frauddetect/SlidingWindowSumObject.java | 53 -
.../frauddetect/TransactionStatsAggregator.java | 99 -
.../demos/frauddetect/TransactionStatsData.java | 36 -
.../operator/HdfsStringOutputOperator.java | 72 -
.../operator/MongoDBOutputOperator.java | 188 -
.../demos/frauddetect/util/JsonUtils.java | 38 -
.../src/main/resources/META-INF/properties.xml | 167 -
.../frauddetect/src/site/conf/my-app-conf1.xml | 27 -
.../frauddetect/FrauddetectApplicationTest.java | 50 -
.../src/test/resources/dt-site-frauddetect.xml | 173 -
.../src/test/resources/log4j.properties | 43 -
demos/highlevelapi/pom.xml | 141 -
demos/highlevelapi/src/assemble/appPackage.xml | 59 -
.../malhar/stream/sample/MinimalWordCount.java | 128 -
.../malhar/stream/sample/WindowedWordCount.java | 290 --
.../stream/sample/complete/AutoComplete.java | 324 --
.../sample/complete/CompletionCandidate.java | 89 -
.../stream/sample/complete/PojoEvent.java | 46 -
.../sample/complete/StreamingWordExtract.java | 162 -
.../stream/sample/complete/TopNByKey.java | 120 -
.../sample/complete/TopWikipediaSessions.java | 347 --
.../stream/sample/complete/TrafficRoutes.java | 523 --
.../sample/complete/TwitterAutoComplete.java | 254 -
.../sample/cookbook/CombinePerKeyExamples.java | 285 --
.../stream/sample/cookbook/DeDupExample.java | 127 -
.../stream/sample/cookbook/InputPojo.java | 78 -
.../sample/cookbook/MaxPerKeyExamples.java | 205 -
.../stream/sample/cookbook/OutputPojo.java | 56 -
.../stream/sample/cookbook/TriggerExample.java | 577 ---
.../src/main/resources/META-INF/properties.xml | 141 -
.../stream/sample/MinimalWordCountTest.java | 61 -
.../stream/sample/WindowedWordCountTest.java | 91 -
.../sample/complete/AutoCompleteTest.java | 62 -
.../complete/StreamingWordExtractTest.java | 144 -
.../complete/TopWikipediaSessionsTest.java | 74 -
.../sample/complete/TrafficRoutesTest.java | 67 -
.../complete/TwitterAutoCompleteTest.java | 66 -
.../cookbook/CombinePerKeyExamplesTest.java | 56 -
.../sample/cookbook/DeDupExampleTest.java | 59 -
.../sample/cookbook/MaxPerKeyExamplesTest.java | 210 -
.../src/test/resources/data/word.txt | 2 -
.../src/test/resources/log4j.properties | 45 -
.../src/test/resources/sampletweets.txt | 44 -
.../src/test/resources/wordcount/word.txt | 8 -
demos/iteration/pom.xml | 37 -
demos/iteration/src/assemble/appPackage.xml | 59 -
.../demos/iteration/Application.java | 171 -
.../demos/iteration/package-info.java | 22 -
.../src/main/resources/META-INF/properties.xml | 45 -
.../demos/iteration/ApplicationTest.java | 86 -
.../src/test/resources/log4j.properties | 43 -
demos/machinedata/pom.xml | 66 -
demos/machinedata/src/assemble/appPackage.xml | 59 -
demos/machinedata/src/main/html/global.js | 269 -
demos/machinedata/src/main/html/index.php | 263 -
demos/machinedata/src/main/html/json.php | 96 -
demos/machinedata/src/main/html/malhar.css | 4688 ------------------
.../demos/machinedata/Application.java | 84 -
.../demos/machinedata/DimensionGenerator.java | 120 -
.../demos/machinedata/InputReceiver.java | 523 --
.../demos/machinedata/data/AverageData.java | 107 -
.../demos/machinedata/data/MachineInfo.java | 158 -
.../demos/machinedata/data/MachineKey.java | 381 --
.../demos/machinedata/data/ResourceType.java | 66 -
.../operator/CalculatorOperator.java | 277 --
.../operator/MachineInfoAveragingOperator.java | 215 -
...chineInfoAveragingPrerequisitesOperator.java | 87 -
.../operator/MachineInfoAveragingUnifier.java | 91 -
.../demos/machinedata/util/Combinatorics.java | 101 -
.../demos/machinedata/util/DataTable.java | 75 -
.../src/main/resources/META-INF/properties.xml | 139 -
.../machinedata/src/site/conf/my-app-conf1.xml | 27 -
.../machinedata/CalculatorOperatorTest.java | 196 -
.../src/test/resources/log4j.properties | 43 -
demos/mobile/pom.xml | 64 -
demos/mobile/src/assemble/appPackage.xml | 59 -
.../datatorrent/demos/mobile/Application.java | 172 -
.../demos/mobile/PhoneEntryOperator.java | 124 -
.../demos/mobile/PhoneMovementGenerator.java | 335 --
.../demos/mobile/doc-files/Mobile.png | Bin 24390 -> 0 bytes
.../datatorrent/demos/mobile/package-info.java | 22 -
.../src/main/resources/META-INF/properties.xml | 82 -
demos/mobile/src/site/conf/my-app-conf1.xml | 27 -
.../demos/mobile/ApplicationTest.java | 117 -
.../src/test/resources/dt-site-mobile.xml | 87 -
.../mobile/src/test/resources/log4j.properties | 43 -
demos/mrmonitor/pom.xml | 64 -
demos/mrmonitor/src/assemble/appPackage.xml | 59 -
.../demos/mrmonitor/Application.java | 56 -
.../datatorrent/demos/mrmonitor/Constants.java | 48 -
.../demos/mrmonitor/MRJobStatusOperator.java | 623 ---
.../mrmonitor/MRMonitoringApplication.java | 93 -
.../demos/mrmonitor/MRStatusObject.java | 501 --
.../com/datatorrent/demos/mrmonitor/MRUtil.java | 99 -
.../demos/mrmonitor/MapToMRObjectOperator.java | 89 -
.../src/main/resources/META-INF/properties.xml | 63 -
.../src/main/resources/mrdebugger.html | 237 -
demos/mrmonitor/src/site/conf/my-app-conf1.xml | 27 -
.../mrmonitor/MrMonitoringApplicationTest.java | 66 -
.../src/test/resources/dt-site-monitoring.xml | 63 -
.../src/test/resources/log4j.properties | 43 -
demos/mroperator/pom.xml | 56 -
demos/mroperator/src/assemble/appPackage.xml | 59 -
.../demos/mroperator/DateWritable.java | 80 -
.../mroperator/HdfsKeyValOutputOperator.java | 41 -
.../mroperator/InvertedIndexApplication.java | 45 -
.../demos/mroperator/LineIndexer.java | 120 -
.../demos/mroperator/LogCountsPerHour.java | 187 -
.../demos/mroperator/LogsCountApplication.java | 46 -
.../demos/mroperator/MapOperator.java | 414 --
.../demos/mroperator/MapReduceApplication.java | 114 -
.../mroperator/NewWordCountApplication.java | 44 -
.../demos/mroperator/OutputCollectorImpl.java | 94 -
.../demos/mroperator/ReduceOperator.java | 189 -
.../demos/mroperator/ReporterImpl.java | 119 -
.../datatorrent/demos/mroperator/WordCount.java | 104 -
.../src/main/resources/META-INF/properties.xml | 88 -
demos/mroperator/src/site/conf/my-app-conf1.xml | 27 -
.../demos/mroperator/MapOperatorTest.java | 159 -
.../demos/mroperator/ReduceOperatorTest.java | 74 -
.../mroperator/WordCountMRApplicationTest.java | 74 -
.../src/test/resources/log4j.properties | 43 -
demos/pi/pom.xml | 45 -
demos/pi/src/assemble/appPackage.xml | 59 -
.../com/datatorrent/demos/pi/Application.java | 92 -
.../demos/pi/ApplicationAppData.java | 135 -
.../demos/pi/ApplicationWithScript.java | 112 -
.../com/datatorrent/demos/pi/Calculator.java | 81 -
.../datatorrent/demos/pi/NamedValueList.java | 94 -
.../demos/pi/PiCalculateOperator.java | 90 -
.../demos/pi/doc-files/Application.gif | Bin 6791 -> 0 bytes
.../demos/pi/doc-files/ApplicationScript.gif | Bin 6687 -> 0 bytes
.../com/datatorrent/demos/pi/package-info.java | 22 -
.../src/main/resources/META-INF/properties.xml | 109 -
.../pi/src/main/resources/PiDemoDataSchema.json | 3 -
demos/pi/src/main/resources/app/PiJsonDemo.json | 52 -
.../datatorrent/demos/pi/ApplicationTest.java | 42 -
.../demos/pi/ApplicationWithScriptTest.java | 35 -
.../datatorrent/demos/pi/CalculatorTest.java | 41 -
demos/pi/src/test/resources/dt-site-pi.xml | 40 -
.../pi/src/test/resources/dt-site-pilibrary.xml | 45 -
demos/pi/src/test/resources/log4j.properties | 43 -
demos/pom.xml | 231 -
demos/r/pom.xml | 83 -
demos/r/src/assemble/appPackage.xml | 59 -
.../demos/r/oldfaithful/FaithfulKey.java | 55 -
.../demos/r/oldfaithful/FaithfulRScript.java | 111 -
.../demos/r/oldfaithful/InputGenerator.java | 145 -
.../r/oldfaithful/OldFaithfulApplication.java | 75 -
.../src/main/resources/META-INF/properties.xml | 41 -
.../demos/r/oldfaithful/eruptionModel.R | 60 -
.../oldfaithful/OldFaithfulApplicationTest.java | 51 -
.../src/test/resources/dt-site-oldfaithful.xml | 41 -
demos/r/src/test/resources/log4j.properties | 43 -
demos/sql/pom.xml | 102 -
demos/sql/src/assemble/appPackage.xml | 59 -
.../sql/sample/FusionStyleSQLApplication.java | 91 -
.../sql/sample/PureStyleSQLApplication.java | 68 -
.../sql/sample/SQLApplicationWithAPI.java | 48 -
.../sql/sample/SQLApplicationWithModelFile.java | 53 -
.../properties-FusionStyleSQLApplication.xml | 65 -
.../properties-PureStyleSQLApplication.xml | 65 -
.../properties-SQLApplicationWithAPI.xml | 43 -
.../properties-SQLApplicationWithModelFile.xml | 32 -
.../src/main/resources/META-INF/properties.xml | 41 -
.../main/resources/model/model_file_csv.json | 27 -
.../sample/FusionStyleSQLApplicationTest.java | 121 -
.../sql/sample/PureStyleSQLApplicationTest.java | 155 -
.../sql/sample/SQLApplicationWithAPITest.java | 92 -
.../sample/SQLApplicationWithModelFileTest.java | 113 -
demos/sql/src/test/resources/input.csv | 6 -
demos/sql/src/test/resources/log4j.properties | 50 -
demos/twitter/pom.xml | 101 -
demos/twitter/src/assemble/appPackage.xml | 59 -
.../twitter/KinesisHashtagsApplication.java | 236 -
.../demos/twitter/SlidingContainer.java | 70 -
.../demos/twitter/TwitterDumpApplication.java | 117 -
.../twitter/TwitterDumpHBaseApplication.java | 81 -
.../twitter/TwitterStatusHashtagExtractor.java | 52 -
.../twitter/TwitterStatusURLExtractor.java | 56 -
.../twitter/TwitterStatusWordExtractor.java | 62 -
.../twitter/TwitterTopCounterApplication.java | 222 -
.../twitter/TwitterTopWordsApplication.java | 71 -
.../TwitterTrendingHashtagsApplication.java | 166 -
.../com/datatorrent/demos/twitter/URLSerDe.java | 74 -
.../demos/twitter/WindowedTopCounter.java | 282 --
.../demos/twitter/doc-files/Application.gif | Bin 8775 -> 0 bytes
.../datatorrent/demos/twitter/package-info.java | 22 -
.../META-INF/properties-TwitterKinesisDemo.xml | 52 -
.../src/main/resources/META-INF/properties.xml | 121 -
demos/twitter/src/main/resources/mysql.sql | 35 -
.../src/main/resources/top_urls.tplg.properties | 48 -
.../resources/twitterHashTagDataSchema.json | 4 -
.../main/resources/twitterURLDataSchema.json | 4 -
.../main/resources/twitterWordDataSchema.json | 4 -
.../twitter/TwitterDumpApplicationTest.java | 48 -
.../demos/twitter/TwitterTopCounterTest.java | 45 -
.../demos/twitter/TwitterTopWordsTest.java | 48 -
.../test/resources/dt-site-rollingtopwords.xml | 73 -
.../twitter/src/test/resources/log4j.properties | 43 -
demos/uniquecount/pom.xml | 50 -
demos/uniquecount/src/assemble/appPackage.xml | 59 -
.../demos/uniquecount/Application.java | 90 -
.../demos/uniquecount/CountVerifier.java | 107 -
.../demos/uniquecount/RandomDataGenerator.java | 115 -
.../demos/uniquecount/RandomKeyValues.java | 153 -
.../demos/uniquecount/RandomKeysGenerator.java | 153 -
.../uniquecount/UniqueKeyValCountDemo.java | 64 -
.../demos/uniquecount/package-info.java | 22 -
.../src/main/resources/META-INF/properties.xml | 29 -
.../uniquecount/src/site/conf/my-app-conf1.xml | 27 -
.../demos/uniquecount/ApplicationTest.java | 38 -
.../demos/uniquecount/UniqueKeyValDemoTest.java | 38 -
.../src/test/resources/log4j.properties | 43 -
demos/wordcount/pom.xml | 49 -
demos/wordcount/src/assemble/appPackage.xml | 59 -
.../demos/wordcount/Application.java | 108 -
.../wordcount/ApplicationWithQuerySupport.java | 133 -
.../demos/wordcount/FileWordCount.java | 349 --
.../datatorrent/demos/wordcount/LineReader.java | 118 -
.../com/datatorrent/demos/wordcount/WCPair.java | 62 -
.../demos/wordcount/WindowWordCount.java | 94 -
.../demos/wordcount/WordCountInputOperator.java | 109 -
.../demos/wordcount/WordCountWriter.java | 109 -
.../datatorrent/demos/wordcount/WordReader.java | 98 -
.../wordcount/doc-files/UniqueWordCounter.jpg | Bin 65015 -> 0 bytes
.../demos/wordcount/package-info.java | 22 -
.../src/main/resources/META-INF/properties.xml | 98 -
.../src/main/resources/WordDataSchema.json | 4 -
.../datatorrent/demos/wordcount/samplefile.txt | 1 -
.../wordcount/src/main/resources/samplefile.txt | 2 -
demos/wordcount/src/site/conf/my-app-conf1.xml | 27 -
.../demos/wordcount/ApplicationTest.java | 51 -
.../src/test/resources/dt-site-wordcount.xml | 37 -
.../src/test/resources/log4j.properties | 43 -
demos/yahoofinance/pom.xml | 65 -
demos/yahoofinance/src/assemble/appPackage.xml | 59 -
.../yahoofinance/ApplicationWithDerbySQL.java | 80 -
.../demos/yahoofinance/StockTickInput.java | 192 -
.../yahoofinance/YahooFinanceApplication.java | 365 --
.../YahooFinanceCSVInputOperator.java | 160 -
.../yahoofinance/doc-files/Application.gif | Bin 12457 -> 0 bytes
.../demos/yahoofinance/package-info.java | 22 -
.../src/main/resources/META-INF/properties.xml | 81 -
.../demos/yahoofinance/ApplicationTest.java | 46 -
.../ApplicationWithDerbySQLTest.java | 50 -
.../src/test/resources/alert_create.json | 24 -
.../src/test/resources/log4j.properties | 43 -
docs/CodingConventionsAndStyle.md | 2 +-
docs/index.md | 2 +-
docs/operators/windowedOperator.md | 2 +-
examples/distributedistinct/pom.xml | 41 +
.../src/assemble/appPackage.xml | 59 +
.../distributeddistinct/Application.java | 69 +
.../distributeddistinct/CountVerifier.java | 103 +
.../IntegerUniqueValueCountAppender.java | 95 +
.../RandomKeyValGenerator.java | 186 +
.../StatefulApplication.java | 76 +
.../UniqueValueCountAppender.java | 236 +
.../src/main/resources/META-INF/properties.xml | 29 +
.../distributeddistinct/ApplicationTest.java | 32 +
.../DistributedDistinctTest.java | 200 +
.../StatefulApplicationTest.java | 78 +
.../StatefulUniqueCountTest.java | 248 +
.../src/test/resources/log4j.properties | 43 +
examples/echoserver/pom.xml | 38 +
examples/echoserver/src/assemble/appPackage.xml | 59 +
.../apex/examples/echoserver/Application.java | 43 +
.../apex/examples/echoserver/Message.java | 30 +
.../examples/echoserver/MessageReceiver.java | 156 +
.../examples/echoserver/MessageResponder.java | 81 +
.../examples/echoserver/NetworkManager.java | 249 +
.../src/main/resources/META-INF/properties.xml | 38 +
.../examples/echoserver/ApplicationTest.java | 53 +
.../src/test/resources/log4j.properties | 43 +
examples/frauddetect/pom.xml | 49 +
.../frauddetect/src/assemble/appPackage.xml | 59 +
.../apex/examples/frauddetect/Application.java | 151 +
.../examples/frauddetect/AverageAlertData.java | 37 +
.../frauddetect/AverageAlertingOperator.java | 186 +
.../frauddetect/BankIdNumberAlertData.java | 36 +
.../examples/frauddetect/BankIdNumberKey.java | 65 +
.../BankIdNumberSamplerOperator.java | 218 +
.../frauddetect/CreditCardAlertData.java | 38 +
.../CreditCardAmountSamplerOperator.java | 206 +
.../examples/frauddetect/CreditCardData.java | 35 +
.../apex/examples/frauddetect/MerchantKey.java | 128 +
.../frauddetect/MerchantQueryInputHandler.java | 69 +
.../frauddetect/MerchantTransaction.java | 202 +
.../MerchantTransactionBucketOperator.java | 148 +
.../MerchantTransactionGenerator.java | 210 +
.../MerchantTransactionInputHandler.java | 116 +
.../frauddetect/SlidingWindowSumKeyVal.java | 99 +
.../frauddetect/SlidingWindowSumObject.java | 53 +
.../frauddetect/TransactionStatsAggregator.java | 99 +
.../frauddetect/TransactionStatsData.java | 36 +
.../operator/HdfsStringOutputOperator.java | 72 +
.../operator/MongoDBOutputOperator.java | 188 +
.../examples/frauddetect/util/JsonUtils.java | 38 +
.../src/main/resources/META-INF/properties.xml | 167 +
.../frauddetect/FrauddetectApplicationTest.java | 50 +
.../src/test/resources/dt-site-frauddetect.xml | 173 +
.../src/test/resources/log4j.properties | 43 +
examples/highlevelapi/pom.xml | 141 +
.../highlevelapi/src/assemble/appPackage.xml | 59 +
.../malhar/stream/sample/MinimalWordCount.java | 128 +
.../malhar/stream/sample/WindowedWordCount.java | 290 ++
.../stream/sample/complete/AutoComplete.java | 324 ++
.../sample/complete/CompletionCandidate.java | 89 +
.../stream/sample/complete/PojoEvent.java | 46 +
.../sample/complete/StreamingWordExtract.java | 162 +
.../stream/sample/complete/TopNByKey.java | 120 +
.../sample/complete/TopWikipediaSessions.java | 347 ++
.../stream/sample/complete/TrafficRoutes.java | 523 ++
.../sample/complete/TwitterAutoComplete.java | 254 +
.../sample/cookbook/CombinePerKeyExamples.java | 285 ++
.../stream/sample/cookbook/DeDupExample.java | 127 +
.../stream/sample/cookbook/InputPojo.java | 78 +
.../sample/cookbook/MaxPerKeyExamples.java | 205 +
.../stream/sample/cookbook/OutputPojo.java | 56 +
.../stream/sample/cookbook/TriggerExample.java | 577 +++
.../src/main/resources/META-INF/properties.xml | 141 +
.../stream/sample/MinimalWordCountTest.java | 61 +
.../stream/sample/WindowedWordCountTest.java | 91 +
.../sample/complete/AutoCompleteTest.java | 62 +
.../complete/StreamingWordExtractTest.java | 144 +
.../complete/TopWikipediaSessionsTest.java | 74 +
.../sample/complete/TrafficRoutesTest.java | 67 +
.../complete/TwitterAutoCompleteTest.java | 66 +
.../cookbook/CombinePerKeyExamplesTest.java | 56 +
.../sample/cookbook/DeDupExampleTest.java | 59 +
.../sample/cookbook/MaxPerKeyExamplesTest.java | 210 +
.../src/test/resources/data/word.txt | 2 +
.../src/test/resources/log4j.properties | 45 +
.../src/test/resources/sampletweets.txt | 44 +
.../src/test/resources/wordcount/word.txt | 8 +
examples/iteration/pom.xml | 37 +
examples/iteration/src/assemble/appPackage.xml | 59 +
.../apex/examples/iteration/Application.java | 171 +
.../apex/examples/iteration/package-info.java | 22 +
.../src/main/resources/META-INF/properties.xml | 45 +
.../examples/iteration/ApplicationTest.java | 86 +
.../src/test/resources/log4j.properties | 43 +
examples/machinedata/pom.xml | 66 +
.../machinedata/src/assemble/appPackage.xml | 59 +
examples/machinedata/src/main/html/global.js | 269 +
examples/machinedata/src/main/html/index.php | 263 +
examples/machinedata/src/main/html/json.php | 96 +
examples/machinedata/src/main/html/malhar.css | 4688 ++++++++++++++++++
.../apex/examples/machinedata/Application.java | 83 +
.../machinedata/DimensionGenerator.java | 121 +
.../examples/machinedata/InputReceiver.java | 524 ++
.../examples/machinedata/data/AverageData.java | 107 +
.../examples/machinedata/data/MachineInfo.java | 158 +
.../examples/machinedata/data/MachineKey.java | 381 ++
.../examples/machinedata/data/ResourceType.java | 66 +
.../operator/CalculatorOperator.java | 279 ++
.../operator/MachineInfoAveragingOperator.java | 216 +
...chineInfoAveragingPrerequisitesOperator.java | 88 +
.../operator/MachineInfoAveragingUnifier.java | 92 +
.../machinedata/util/Combinatorics.java | 101 +
.../examples/machinedata/util/DataTable.java | 75 +
.../src/main/resources/META-INF/properties.xml | 139 +
.../machinedata/CalculatorOperatorTest.java | 197 +
.../src/test/resources/log4j.properties | 43 +
examples/mobile/pom.xml | 64 +
examples/mobile/src/assemble/appPackage.xml | 59 +
.../apex/examples/mobile/Application.java | 172 +
.../examples/mobile/PhoneEntryOperator.java | 124 +
.../examples/mobile/PhoneMovementGenerator.java | 335 ++
.../apex/examples/mobile/doc-files/Mobile.png | Bin 0 -> 24390 bytes
.../apex/examples/mobile/package-info.java | 22 +
.../src/main/resources/META-INF/properties.xml | 82 +
.../apex/examples/mobile/ApplicationTest.java | 117 +
.../src/test/resources/dt-site-mobile.xml | 87 +
.../mobile/src/test/resources/log4j.properties | 43 +
examples/mrmonitor/pom.xml | 64 +
examples/mrmonitor/src/assemble/appPackage.xml | 59 +
.../apex/examples/mrmonitor/Application.java | 56 +
.../apex/examples/mrmonitor/Constants.java | 48 +
.../examples/mrmonitor/MRJobStatusOperator.java | 622 +++
.../mrmonitor/MRMonitoringApplication.java | 93 +
.../apex/examples/mrmonitor/MRStatusObject.java | 501 ++
.../apache/apex/examples/mrmonitor/MRUtil.java | 99 +
.../mrmonitor/MapToMRObjectOperator.java | 89 +
.../src/main/resources/META-INF/properties.xml | 63 +
.../src/main/resources/mrdebugger.html | 237 +
.../mrmonitor/MrMonitoringApplicationTest.java | 66 +
.../src/test/resources/dt-site-monitoring.xml | 63 +
.../src/test/resources/log4j.properties | 43 +
examples/mroperator/pom.xml | 56 +
examples/mroperator/src/assemble/appPackage.xml | 59 +
.../apex/examples/mroperator/DateWritable.java | 80 +
.../mroperator/HdfsKeyValOutputOperator.java | 41 +
.../mroperator/InvertedIndexApplication.java | 45 +
.../apex/examples/mroperator/LineIndexer.java | 120 +
.../examples/mroperator/LogCountsPerHour.java | 187 +
.../mroperator/LogsCountApplication.java | 46 +
.../apex/examples/mroperator/MapOperator.java | 414 ++
.../mroperator/MapReduceApplication.java | 114 +
.../mroperator/NewWordCountApplication.java | 44 +
.../mroperator/OutputCollectorImpl.java | 94 +
.../examples/mroperator/ReduceOperator.java | 188 +
.../apex/examples/mroperator/ReporterImpl.java | 119 +
.../apex/examples/mroperator/WordCount.java | 104 +
.../src/main/resources/META-INF/properties.xml | 88 +
.../examples/mroperator/MapOperatorTest.java | 159 +
.../examples/mroperator/ReduceOperatorTest.java | 74 +
.../mroperator/WordCountMRApplicationTest.java | 74 +
.../src/test/resources/log4j.properties | 43 +
examples/pi/pom.xml | 45 +
examples/pi/src/assemble/appPackage.xml | 59 +
.../apache/apex/examples/pi/Application.java | 92 +
.../apex/examples/pi/ApplicationAppData.java | 135 +
.../apex/examples/pi/ApplicationWithScript.java | 112 +
.../org/apache/apex/examples/pi/Calculator.java | 81 +
.../apache/apex/examples/pi/NamedValueList.java | 94 +
.../apex/examples/pi/PiCalculateOperator.java | 90 +
.../apex/examples/pi/doc-files/Application.gif | Bin 0 -> 6791 bytes
.../examples/pi/doc-files/ApplicationScript.gif | Bin 0 -> 6687 bytes
.../apache/apex/examples/pi/package-info.java | 22 +
.../src/main/resources/META-INF/properties.xml | 109 +
.../src/main/resources/PiExampleDataSchema.json | 3 +
.../src/main/resources/app/PiJsonExample.json | 52 +
.../apex/examples/pi/ApplicationTest.java | 42 +
.../examples/pi/ApplicationWithScriptTest.java | 35 +
.../apache/apex/examples/pi/CalculatorTest.java | 41 +
examples/pi/src/test/resources/dt-site-pi.xml | 40 +
.../pi/src/test/resources/dt-site-pilibrary.xml | 45 +
examples/pi/src/test/resources/log4j.properties | 43 +
examples/pom.xml | 231 +
examples/r/pom.xml | 83 +
examples/r/src/assemble/appPackage.xml | 59 +
.../examples/r/oldfaithful/FaithfulKey.java | 55 +
.../examples/r/oldfaithful/FaithfulRScript.java | 111 +
.../examples/r/oldfaithful/InputGenerator.java | 145 +
.../r/oldfaithful/OldFaithfulApplication.java | 75 +
.../src/main/resources/META-INF/properties.xml | 41 +
.../apex/examples/r/oldfaithful/eruptionModel.R | 60 +
.../oldfaithful/OldFaithfulApplicationTest.java | 51 +
.../src/test/resources/dt-site-oldfaithful.xml | 41 +
examples/r/src/test/resources/log4j.properties | 43 +
examples/sql/pom.xml | 102 +
examples/sql/src/assemble/appPackage.xml | 59 +
.../sql/sample/FusionStyleSQLApplication.java | 91 +
.../sql/sample/PureStyleSQLApplication.java | 68 +
.../sql/sample/SQLApplicationWithAPI.java | 48 +
.../sql/sample/SQLApplicationWithModelFile.java | 53 +
.../properties-FusionStyleSQLApplication.xml | 65 +
.../properties-PureStyleSQLApplication.xml | 65 +
.../properties-SQLApplicationWithAPI.xml | 43 +
.../properties-SQLApplicationWithModelFile.xml | 32 +
.../src/main/resources/META-INF/properties.xml | 41 +
.../main/resources/model/model_file_csv.json | 27 +
.../sample/FusionStyleSQLApplicationTest.java | 121 +
.../sql/sample/PureStyleSQLApplicationTest.java | 155 +
.../sql/sample/SQLApplicationWithAPITest.java | 92 +
.../sample/SQLApplicationWithModelFileTest.java | 113 +
examples/sql/src/test/resources/input.csv | 6 +
.../sql/src/test/resources/log4j.properties | 50 +
examples/twitter/pom.xml | 101 +
examples/twitter/src/assemble/appPackage.xml | 59 +
.../twitter/KinesisHashtagsApplication.java | 236 +
.../apex/examples/twitter/SlidingContainer.java | 70 +
.../twitter/TwitterDumpApplication.java | 117 +
.../twitter/TwitterDumpHBaseApplication.java | 81 +
.../twitter/TwitterStatusHashtagExtractor.java | 52 +
.../twitter/TwitterStatusURLExtractor.java | 56 +
.../twitter/TwitterStatusWordExtractor.java | 62 +
.../twitter/TwitterTopCounterApplication.java | 222 +
.../twitter/TwitterTopWordsApplication.java | 71 +
.../TwitterTrendingHashtagsApplication.java | 166 +
.../apache/apex/examples/twitter/URLSerDe.java | 74 +
.../examples/twitter/WindowedTopCounter.java | 282 ++
.../examples/twitter/doc-files/Application.gif | Bin 0 -> 8775 bytes
.../apex/examples/twitter/package-info.java | 22 +
.../META-INF/properties-TwitterKinesisDemo.xml | 52 +
.../src/main/resources/META-INF/properties.xml | 121 +
examples/twitter/src/main/resources/mysql.sql | 35 +
.../src/main/resources/top_urls.tplg.properties | 48 +
.../resources/twitterHashTagDataSchema.json | 4 +
.../main/resources/twitterURLDataSchema.json | 4 +
.../main/resources/twitterWordDataSchema.json | 4 +
.../twitter/TwitterDumpApplicationTest.java | 48 +
.../examples/twitter/TwitterTopCounterTest.java | 45 +
.../examples/twitter/TwitterTopWordsTest.java | 48 +
.../test/resources/dt-site-rollingtopwords.xml | 73 +
.../twitter/src/test/resources/log4j.properties | 43 +
examples/uniquecount/pom.xml | 50 +
.../uniquecount/src/assemble/appPackage.xml | 59 +
.../apex/examples/uniquecount/Application.java | 90 +
.../examples/uniquecount/CountVerifier.java | 107 +
.../uniquecount/RandomDataGenerator.java | 115 +
.../examples/uniquecount/RandomKeyValues.java | 153 +
.../uniquecount/RandomKeysGenerator.java | 153 +
.../uniquecount/UniqueKeyValCountExample.java | 64 +
.../apex/examples/uniquecount/package-info.java | 22 +
.../src/main/resources/META-INF/properties.xml | 29 +
.../examples/uniquecount/ApplicationTest.java | 38 +
.../uniquecount/UniqueKeyValExampleTest.java | 38 +
.../src/test/resources/log4j.properties | 43 +
examples/wordcount/pom.xml | 49 +
examples/wordcount/src/assemble/appPackage.xml | 59 +
.../apex/examples/wordcount/Application.java | 108 +
.../wordcount/ApplicationWithQuerySupport.java | 133 +
.../apex/examples/wordcount/FileWordCount.java | 349 ++
.../apex/examples/wordcount/LineReader.java | 118 +
.../apache/apex/examples/wordcount/WCPair.java | 62 +
.../examples/wordcount/WindowWordCount.java | 94 +
.../wordcount/WordCountInputOperator.java | 109 +
.../examples/wordcount/WordCountWriter.java | 109 +
.../apex/examples/wordcount/WordReader.java | 98 +
.../wordcount/doc-files/UniqueWordCounter.jpg | Bin 0 -> 65015 bytes
.../apex/examples/wordcount/package-info.java | 22 +
.../src/main/resources/META-INF/properties.xml | 98 +
.../src/main/resources/WordDataSchema.json | 4 +
.../apex/examples/wordcount/samplefile.txt | 1 +
.../wordcount/src/main/resources/samplefile.txt | 2 +
.../examples/wordcount/ApplicationTest.java | 51 +
.../src/test/resources/dt-site-wordcount.xml | 37 +
.../src/test/resources/log4j.properties | 43 +
examples/yahoofinance/pom.xml | 65 +
.../yahoofinance/src/assemble/appPackage.xml | 59 +
.../yahoofinance/ApplicationWithDerbySQL.java | 80 +
.../examples/yahoofinance/StockTickInput.java | 192 +
.../yahoofinance/YahooFinanceApplication.java | 365 ++
.../YahooFinanceCSVInputOperator.java | 160 +
.../yahoofinance/doc-files/Application.gif | Bin 0 -> 12457 bytes
.../examples/yahoofinance/package-info.java | 22 +
.../src/main/resources/META-INF/properties.xml | 81 +
.../examples/yahoofinance/ApplicationTest.java | 46 +
.../ApplicationWithDerbySQLTest.java | 50 +
.../src/test/resources/alert_create.json | 24 +
.../src/test/resources/log4j.properties | 43 +
.../test/resources/SocketInputOperatorTest.txt | 2 +-
pom.xml | 2 +-
581 files changed, 33391 insertions(+), 33626 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 4781031..4016151 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,7 @@
Apache Apex Malhar
===============================
-Malhar repository contains open source operator and codec library that can be used with the Apache Apex platform to build Realtime streaming applications. In addition to the library there are benchmark, contrib, demos, webdemos and samples folders available. Demos contain demo applications built using the library operators. Webdemos contain webpages for the demos. Benchmark contains performance testing applications. Contrib contains additional operators that interface with third party softwares. Samples contain some sample code that shows how to use the library operators.
+Malhar repository contains open source operator and codec library that can be used with the Apache Apex platform to build Realtime streaming applications. In addition to the library there are benchmark, contrib, examples and samples folders available. Examples contain applications built using the library operators. Benchmark contains performance testing applications. Contrib contains additional operators that interface with third party softwares. Samples contain some sample code that shows how to use the library operators.
Contributing
------------
@@ -32,7 +32,7 @@ mvn install -DskipTests
The code depends on the Apex API, which is available in the Maven Central Repository. Individual modules such as library can be built independently by changing directory to the module and running maven there.
-Multiple [demo applications](demos/) are provided to showcase the Apex platform and application development process.
+Multiple [example applications](examples/) are provided to showcase the Apex platform and application development process.
License
--------------------
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/distributedistinct/pom.xml
----------------------------------------------------------------------
diff --git a/demos/distributedistinct/pom.xml b/demos/distributedistinct/pom.xml
deleted file mode 100644
index 3bb4400..0000000
--- a/demos/distributedistinct/pom.xml
+++ /dev/null
@@ -1,41 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>distributedistinct</artifactId>
- <packaging>jar</packaging>
-
- <name>Apache Apex Malhar Distributed Distinct Demo</name>
- <description></description>
-
- <parent>
- <groupId>org.apache.apex</groupId>
- <artifactId>malhar-demos</artifactId>
- <version>3.7.0-SNAPSHOT</version>
- </parent>
-
- <properties>
- <skipTests>true</skipTests>
- </properties>
-
-</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/distributedistinct/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/demos/distributedistinct/src/assemble/appPackage.xml b/demos/distributedistinct/src/assemble/appPackage.xml
deleted file mode 100644
index 4138cf2..0000000
--- a/demos/distributedistinct/src/assemble/appPackage.xml
+++ /dev/null
@@ -1,59 +0,0 @@
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
- <id>appPackage</id>
- <formats>
- <format>jar</format>
- </formats>
- <includeBaseDirectory>false</includeBaseDirectory>
- <fileSets>
- <fileSet>
- <directory>${basedir}/target/</directory>
- <outputDirectory>/app</outputDirectory>
- <includes>
- <include>${project.artifactId}-${project.version}.jar</include>
- </includes>
- </fileSet>
- <fileSet>
- <directory>${basedir}/target/deps</directory>
- <outputDirectory>/lib</outputDirectory>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/site/conf</directory>
- <outputDirectory>/conf</outputDirectory>
- <includes>
- <include>*.xml</include>
- </includes>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/main/resources/META-INF</directory>
- <outputDirectory>/META-INF</outputDirectory>
- </fileSet>
- <fileSet>
- <directory>${basedir}/src/main/resources/app</directory>
- <outputDirectory>/app</outputDirectory>
- </fileSet>
- </fileSets>
-
-</assembly>
-
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/distributedistinct/src/main/java/com/datatorrent/demos/distributeddistinct/Application.java
----------------------------------------------------------------------
diff --git a/demos/distributedistinct/src/main/java/com/datatorrent/demos/distributeddistinct/Application.java b/demos/distributedistinct/src/main/java/com/datatorrent/demos/distributeddistinct/Application.java
deleted file mode 100644
index 656b083..0000000
--- a/demos/distributedistinct/src/main/java/com/datatorrent/demos/distributeddistinct/Application.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.distributeddistinct;
-
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-import com.datatorrent.lib.algo.UniqueValueCount;
-import com.datatorrent.lib.io.ConsoleOutputOperator;
-import com.datatorrent.lib.stream.Counter;
-import com.datatorrent.lib.stream.StreamDuplicater;
-import com.datatorrent.lib.util.KeyValPair;
-
-/**
- * This application demonstrates the UniqueValueCount operator. It uses an input operator which generates random key
- * value pairs and simultaneously emits them to the UniqueValueCount operator and keeps track of the number of unique
- * values per key to emit to the verifier.
- *
- * @since 1.0.4
- */
-@ApplicationAnnotation(name = "ValueCount")
-public class Application implements StreamingApplication
-{
-
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- RandomKeyValGenerator randGen = dag.addOperator("RandomGenerator", new RandomKeyValGenerator());
- UniqueValueCount<Integer> valCount = dag.addOperator("UniqueCounter", new UniqueValueCount<Integer>());
- ConsoleOutputOperator consOut = dag.addOperator("Console", new ConsoleOutputOperator());
- StreamDuplicater<KeyValPair<Integer, Integer>> dup = dag.addOperator("Duplicator", new StreamDuplicater<KeyValPair<Integer, Integer>>());
- CountVerifier verifier = dag.addOperator("Verifier", new CountVerifier());
- ConsoleOutputOperator successOutput = dag.addOperator("Success", new ConsoleOutputOperator());
- successOutput.setStringFormat("Success %d");
- ConsoleOutputOperator failureOutput = dag.addOperator("Failure", new ConsoleOutputOperator());
- failureOutput.setStringFormat("Failure %d");
-
- Counter successcounter = dag.addOperator("SuccessCounter", new Counter());
- Counter failurecounter = dag.addOperator("FailureCounter", new Counter());
-
- dag.addStream("Events", randGen.outport, valCount.input);
- dag.addStream("Duplicates", valCount.output, dup.data);
- dag.addStream("Unverified", dup.out1, verifier.recIn);
- dag.addStream("EventCount", randGen.verport, verifier.trueIn);
- dag.addStream("Verified", verifier.successPort, successcounter.input);
- dag.addStream("Failed", verifier.failurePort, failurecounter.input);
- dag.addStream("SuccessCount", successcounter.output, successOutput.input);
- dag.addStream("FailedCount", failurecounter.output, failureOutput.input);
- dag.addStream("Output", dup.out2, consOut.input);
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/distributedistinct/src/main/java/com/datatorrent/demos/distributeddistinct/CountVerifier.java
----------------------------------------------------------------------
diff --git a/demos/distributedistinct/src/main/java/com/datatorrent/demos/distributeddistinct/CountVerifier.java b/demos/distributedistinct/src/main/java/com/datatorrent/demos/distributeddistinct/CountVerifier.java
deleted file mode 100644
index 417ed7c..0000000
--- a/demos/distributedistinct/src/main/java/com/datatorrent/demos/distributeddistinct/CountVerifier.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.distributeddistinct;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import com.datatorrent.api.Context.OperatorContext;
-import com.datatorrent.api.DefaultInputPort;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.api.Operator;
-import com.datatorrent.api.annotation.OutputPortFieldAnnotation;
-import com.datatorrent.lib.util.KeyValPair;
-
-/**
- * An operator that takes two streams of Integer to Integer KeyValPairs and verifies that the two streams output the
- * same KeyValPairs within a given window.
- *
- * @since 1.0.4
- */
-public class CountVerifier implements Operator
-{
- Map<Integer, Integer> trueCount = new HashMap<Integer, Integer>();
- Map<Integer, Integer> receivedCount = new HashMap<Integer, Integer>();
-
- public final transient DefaultInputPort<KeyValPair<Integer, Integer>> trueIn = new DefaultInputPort<KeyValPair<Integer, Integer>>()
- {
- @Override
- public void process(KeyValPair<Integer, Integer> tuple)
- {
- trueCount.put(tuple.getKey(), tuple.getValue());
- }
- };
-
- public final transient DefaultInputPort<KeyValPair<Integer, Integer>> recIn = new DefaultInputPort<KeyValPair<Integer, Integer>>()
- {
- @Override
- public void process(KeyValPair<Integer, Integer> tuple)
- {
- receivedCount.put(tuple.getKey(), tuple.getValue());
- }
- };
-
- @OutputPortFieldAnnotation(optional = true)
- public final transient DefaultOutputPort<Integer> successPort = new DefaultOutputPort<Integer>();
- @OutputPortFieldAnnotation(optional = true)
- public final transient DefaultOutputPort<Integer> failurePort = new DefaultOutputPort<Integer>();
-
- @Override
- public void setup(OperatorContext arg0)
- {
-
- }
-
- @Override
- public void teardown()
- {
-
- }
-
- @Override
- public void beginWindow(long windowID)
- {
- }
-
- /**
- * Checks that the key to value pairs are the same and counts the number of pairs that are different. If there are
- * failures, it will emit the number of failures to the failure port. Otherwise, it will emit the number of keys to
- * the success port.
- */
- @Override
- public void endWindow()
- {
- int failureCount = 0;
- for (Map.Entry<Integer, Integer> e : receivedCount.entrySet()) {
- Integer key = e.getKey();
- if (!trueCount.get(key).equals(e.getValue())) {
- failureCount++;
- }
- }
- if (failureCount != 0) {
- failurePort.emit(failureCount);
- } else {
- successPort.emit(trueCount.size());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/distributedistinct/src/main/java/com/datatorrent/demos/distributeddistinct/IntegerUniqueValueCountAppender.java
----------------------------------------------------------------------
diff --git a/demos/distributedistinct/src/main/java/com/datatorrent/demos/distributeddistinct/IntegerUniqueValueCountAppender.java b/demos/distributedistinct/src/main/java/com/datatorrent/demos/distributeddistinct/IntegerUniqueValueCountAppender.java
deleted file mode 100644
index bf8a307..0000000
--- a/demos/distributedistinct/src/main/java/com/datatorrent/demos/distributeddistinct/IntegerUniqueValueCountAppender.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.distributeddistinct;
-
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.HashSet;
-import java.util.Set;
-
-import com.datatorrent.lib.algo.UniqueValueCount;
-import com.datatorrent.lib.util.KeyValPair;
-
-/**
- * This operator demonstrates {@link UniqueValueCountAppender} given that the keys and values of the preceding {@link UniqueValueCount} operator
- * are both integers. <br/>
- * It will keep track of the number of all the unique values emitted per key since the application starts.
- *
- * @since 1.0.4
- */
-public class IntegerUniqueValueCountAppender extends UniqueValueCountAppender<Integer>
-{
- @Override
- public Object processResultSet(ResultSet resultSet)
- {
- Set<Integer> valSet = new HashSet<Integer>();
- try {
- while (resultSet.next()) {
- valSet.add(resultSet.getInt(1));
- }
- return valSet;
- } catch (SQLException e) {
- throw new RuntimeException("while processing the result set", e);
- }
- }
-
- @Override
- protected void prepareGetStatement(PreparedStatement getStatement, Object key) throws SQLException
- {
- getStatement.setInt(1, (Integer)key);
- }
-
- @Override
- protected void preparePutStatement(PreparedStatement putStatement, Object key, Object value) throws SQLException
- {
- @SuppressWarnings("unchecked")
- Set<Integer> valueSet = (Set<Integer>)value;
- for (Integer val : valueSet) {
- @SuppressWarnings("unchecked")
- Set<Integer> currentVals = (Set<Integer>)get(key);
- if (!currentVals.contains(val)) {
- batch = true;
- putStatement.setInt(1, (Integer)key);
- putStatement.setInt(2, val);
- putStatement.setLong(3, windowID);
- putStatement.addBatch();
- }
- }
- }
-
- @Override
- public void endWindow()
- {
- try {
- Statement stmt = store.getConnection().createStatement();
- String keySetQuery = "SELECT DISTINCT col1 FROM " + tableName;
- ResultSet resultSet = stmt.executeQuery(keySetQuery);
- while (resultSet.next()) {
- int val = resultSet.getInt(1);
- @SuppressWarnings("unchecked")
- Set<Integer> valSet = (Set<Integer>)cacheManager.get(val);
- output.emit(new KeyValPair<Object, Object>(val, valSet.size()));
- }
- } catch (SQLException e) {
- throw new RuntimeException("While emitting tuples", e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/distributedistinct/src/main/java/com/datatorrent/demos/distributeddistinct/RandomKeyValGenerator.java
----------------------------------------------------------------------
diff --git a/demos/distributedistinct/src/main/java/com/datatorrent/demos/distributeddistinct/RandomKeyValGenerator.java b/demos/distributedistinct/src/main/java/com/datatorrent/demos/distributeddistinct/RandomKeyValGenerator.java
deleted file mode 100644
index c8016da..0000000
--- a/demos/distributedistinct/src/main/java/com/datatorrent/demos/distributeddistinct/RandomKeyValGenerator.java
+++ /dev/null
@@ -1,186 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.distributeddistinct;
-
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Random;
-import java.util.Set;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.datatorrent.api.Context.OperatorContext;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.api.InputOperator;
-import com.datatorrent.api.annotation.OutputPortFieldAnnotation;
-import com.datatorrent.lib.util.KeyHashValPair;
-import com.datatorrent.lib.util.KeyValPair;
-
-/**
- * Generates random KeyValPairs and optionally, keeps track of the number of unique values per pair to emit to the
- * verification port.
- *
- * @since 1.0.4
- */
-public class RandomKeyValGenerator implements InputOperator
-{
-
- protected int numKeys = 50;
- protected int numVals = 1000;
- protected int tupleBlast = 1000;
- protected Map<Integer, Set<Integer>> valhistory = new HashMap<Integer, Set<Integer>>();
- private Random rand = new Random();
- private boolean once;
- private boolean clearHistory;
- @OutputPortFieldAnnotation(optional = false)
- public transient DefaultOutputPort<KeyValPair<Integer, Object>> outport = new DefaultOutputPort<KeyValPair<Integer, Object>>();
-
- @OutputPortFieldAnnotation(optional = true)
- public transient DefaultOutputPort<KeyValPair<Integer, Integer>> verport = new DefaultOutputPort<KeyValPair<Integer, Integer>>();
-
- /**
- * Ensures that the generator emits KeyValPairs once per window
- */
- @Override
- public void beginWindow(long l)
- {
- once = false;
- }
-
- /**
- * Emits the total count of unique values per key as KeyHashValPairs to the verification port
- */
- @Override
- public void endWindow()
- {
- if (verport.isConnected()) {
- for (Map.Entry<Integer, Set<Integer>> e : valhistory.entrySet()) {
- verport.emit(new KeyHashValPair<Integer, Integer>(e.getKey(), e.getValue().size()));
- }
- }
- if (clearHistory) {
- valhistory.clear();
- }
- }
-
- @Override
- public void setup(OperatorContext arg0)
- {
-
- }
-
- @Override
- public void teardown()
- {
-
- }
-
- /**
- * Emits random KeyValPairs and keeps track of the unique values per key.
- */
- @Override
- public void emitTuples()
- {
- if (!once) {
- int key;
- int val;
- for (int i = 0; i < tupleBlast; i++) {
- key = rand.nextInt(numKeys);
- val = rand.nextInt(numVals);
- outport.emit(new KeyValPair<Integer, Object>(key, val));
- if (verport.isConnected()) {
- Set<Integer> count = valhistory.get(key);
- if (count == null) {
- Set<Integer> tempset = new HashSet<Integer>();
- tempset.add(val);
- valhistory.put(key, tempset);
- LOG.debug("key {} val {}", key, tempset);
- } else if (!valhistory.get(key).contains(val)) {
- valhistory.get(key).add(val);
- }
- }
- }
- once = true;
- }
- }
-
- /**
- * @return the number of possible keys
- */
- public int getNumKeys()
- {
- return numKeys;
- }
-
- /**
- * Sets the number of possible keys to numKeys
- *
- * @param numKeys
- * the new number of possible keys
- */
- public void setNumKeys(int numKeys)
- {
- this.numKeys = numKeys;
- }
-
- /**
- * Returns the number of possible values that can be emitted
- *
- * @return the number of possible values that can be emitted
- */
- public int getNumVals()
- {
- return numVals;
- }
-
- /**
- * Sets the number of possible values that can be emitted to numVals
- *
- * @param numVals
- * the number of possible values that can be emitted
- */
- public void setNumVals(int numVals)
- {
- this.numVals = numVals;
- }
-
- /**
- * Sets the number of KeyValPairs to be emitted to tupleBlast
- *
- * @param tupleBlast
- * the new number of KeyValPairs to be emitted
- */
- public void setTupleBlast(int tupleBlast)
- {
- this.tupleBlast = tupleBlast;
- }
-
- /**
- * tuple blast
- * @return
- */
- public int getTupleBlast()
- {
- return tupleBlast;
- }
-
- private static final Logger LOG = LoggerFactory.getLogger(RandomKeyValGenerator.class);
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/distributedistinct/src/main/java/com/datatorrent/demos/distributeddistinct/StatefulApplication.java
----------------------------------------------------------------------
diff --git a/demos/distributedistinct/src/main/java/com/datatorrent/demos/distributeddistinct/StatefulApplication.java b/demos/distributedistinct/src/main/java/com/datatorrent/demos/distributeddistinct/StatefulApplication.java
deleted file mode 100644
index d6c488c..0000000
--- a/demos/distributedistinct/src/main/java/com/datatorrent/demos/distributeddistinct/StatefulApplication.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.distributeddistinct;
-
-import java.io.Serializable;
-
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.Context;
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-import com.datatorrent.lib.algo.UniqueValueCount;
-import com.datatorrent.lib.algo.UniqueValueCount.InternalCountOutput;
-import com.datatorrent.lib.codec.KryoSerializableStreamCodec;
-import com.datatorrent.lib.io.ConsoleOutputOperator;
-
-/**
- * This application demonstrates the UniqueValueCount operator. It uses an input operator which generates random key
- * value pairs and emits them to the UniqueValueCount operator which keeps track of the unique values per window. It
- * then emits the values to the StatefulUniqueCount which uses a combination of a cache and database to keep track of
- * the overall unique values and outputs the resulting unique value count to the ConsoleOutputOperator.
- *
- * @since 1.0.4
- */
-@ApplicationAnnotation(name = "StatefulDistinctCount")
-public class StatefulApplication implements StreamingApplication
-{
- @SuppressWarnings("unchecked")
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- RandomKeyValGenerator randGen = dag.addOperator("RandomGenerator", new RandomKeyValGenerator());
- UniqueValueCount<Integer> valCount = dag.addOperator("UniqueCounter", new UniqueValueCount<Integer>());
- ConsoleOutputOperator consOut = dag.addOperator("Console", new ConsoleOutputOperator());
- IntegerUniqueValueCountAppender uniqueUnifier = dag.addOperator("StatefulUniqueCounter", new IntegerUniqueValueCountAppender());
- dag.getOperatorMeta("StatefulUniqueCounter").getMeta(uniqueUnifier.input).getAttributes().put(Context.PortContext.STREAM_CODEC, new KeyBasedStreamCodec());
-
- @SuppressWarnings("rawtypes")
- DefaultOutputPort valOut = valCount.output;
- @SuppressWarnings("rawtypes")
- DefaultOutputPort uniqueOut = uniqueUnifier.output;
-
- dag.addStream("Events", randGen.outport, valCount.input);
- dag.addStream("Unified", valOut, uniqueUnifier.input);
- dag.addStream("Result", uniqueOut, consOut.input);
- }
-
- public static class KeyBasedStreamCodec extends KryoSerializableStreamCodec<InternalCountOutput<Integer>> implements Serializable
- {
- @Override
- public int getPartition(InternalCountOutput<Integer> t)
- {
- return t.getKey().hashCode();
- }
-
- private static final long serialVersionUID = 201407231527L;
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/distributedistinct/src/main/java/com/datatorrent/demos/distributeddistinct/UniqueValueCountAppender.java
----------------------------------------------------------------------
diff --git a/demos/distributedistinct/src/main/java/com/datatorrent/demos/distributeddistinct/UniqueValueCountAppender.java b/demos/distributedistinct/src/main/java/com/datatorrent/demos/distributeddistinct/UniqueValueCountAppender.java
deleted file mode 100644
index 3f14e0f..0000000
--- a/demos/distributedistinct/src/main/java/com/datatorrent/demos/distributeddistinct/UniqueValueCountAppender.java
+++ /dev/null
@@ -1,236 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.distributeddistinct;
-
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Map;
-import java.util.Set;
-
-import javax.annotation.Nonnull;
-import javax.validation.constraints.Min;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-
-import com.datatorrent.api.Context;
-import com.datatorrent.api.DefaultPartition;
-import com.datatorrent.api.Partitioner;
-import com.datatorrent.lib.algo.UniqueValueCount;
-import com.datatorrent.lib.algo.UniqueValueCount.InternalCountOutput;
-import com.datatorrent.lib.db.jdbc.JDBCLookupCacheBackedOperator;
-import com.datatorrent.netlet.util.DTThrowable;
-
-/**
- * <p>
- * This operator supplements the {@link UniqueValueCount} operator by making it state-full.<br/>
- * It helps to track unique values through out the lifetime of the application.
- * </p>
- *
- * <p>
- * The operator counts the number of values emitted per key by storing previously
- * counted values in both a transient cache and in a persistent database. <br/>
- * In case of a rollback, it will erase all values committed to the database
- * in the windows greater than the activation window, then re-enter them as needed to keep it stateful.<br/>
- * This operator, when appended to {@link UniqueValueCount} will keep track of the
- * unique values emitted since the start of the application.
- *
- * @since 1.0.4
- */
-public abstract class UniqueValueCountAppender<V> extends JDBCLookupCacheBackedOperator<InternalCountOutput<V>> implements Partitioner<UniqueValueCountAppender<V>>
-{
- protected Set<Integer> partitionKeys;
- protected int partitionMask;
- protected transient long windowID;
- protected transient boolean batch;
- @Min(1)
- private int partitionCount = 1;
-
- public UniqueValueCountAppender()
-
- {
- partitionKeys = Sets.newHashSet(0);
- partitionMask = 0;
- }
-
- public void setPartitionCount(int partitionCount)
- {
- this.partitionCount = partitionCount;
- }
-
- public int getPartitionCount()
- {
- return partitionCount;
- }
-
- @Override
- public void setup(Context.OperatorContext context)
- {
- super.setup(context);
- LOGGER.debug("store properties {} {}", store.getDatabaseDriver(), store.getDatabaseUrl());
- LOGGER.debug("table name {}", tableName);
- windowID = context.getValue(Context.OperatorContext.ACTIVATION_WINDOW_ID);
- try {
- ResultSet resultSet = store.getConnection().createStatement().executeQuery("SELECT col1 FROM " + tableName + " WHERE col3 >= " + windowID);
- PreparedStatement deleteStatement = store.getConnection().prepareStatement("DELETE FROM " + tableName + " WHERE col3 >= " + windowID + " AND col1 = ?");
-
- Set<Object> deletedKeys = Sets.newHashSet();
- while (resultSet.next()) {
- Object key = resultSet.getObject(1);
- if (partitionKeys.contains((key.hashCode() & partitionMask)) && !deletedKeys.contains(key)) {
- deletedKeys.add(key);
- deleteStatement.setObject(1, key);
- deleteStatement.executeUpdate();
- }
- }
- } catch (SQLException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- protected void processTuple(InternalCountOutput<V> tuple)
- {
-
- Object key = getKeyFromTuple(tuple);
- @SuppressWarnings("unchecked")
- Set<Object> values = (Set<Object>)cacheManager.get(key);
- if (values == null) {
- values = Sets.newHashSet();
- }
- values.addAll(tuple.getInternalSet());
- cacheManager.put(key, values);
- }
-
- @Override
- protected String fetchInsertQuery()
- {
- return "INSERT INTO " + tableName + " (col1, col2, col3) VALUES (?, ?, ?)";
- }
-
- @Override
- protected String fetchGetQuery()
- {
- return "select col2 from " + tableName + " where col1 = ?";
- }
-
- @Override
- public Map<Object, Object> loadInitialData()
- {
- return null;
- }
-
- @Override
- public void put(@Nonnull Object key, @Nonnull Object value)
- {
- try {
- batch = false;
- preparePutStatement(putStatement, key, value);
- if (batch) {
- putStatement.executeBatch();
- putStatement.clearBatch();
- }
- } catch (SQLException e) {
- throw new RuntimeException("while executing insert", e);
- }
- }
-
- @Override
- public void teardown()
- {
-
- }
-
- @Override
- public void beginWindow(long windowID)
- {
- this.windowID = windowID;
- }
-
- @Override
- protected Object getKeyFromTuple(InternalCountOutput<V> tuple)
- {
- return tuple.getKey();
- }
-
- @Override
- public void putAll(Map<Object, Object> m)
- {
- throw new UnsupportedOperationException("not supported");
- }
-
- @Override
- public void remove(Object key)
- {
- throw new UnsupportedOperationException("not supported");
- }
-
- /**
- * Assigns the partitions according to certain key values and keeps track of the
- * keys that each partition will be processing so that in the case of a
- * rollback, each partition will only clear the data that it is responsible for.
- */
- @Override
- public Collection<com.datatorrent.api.Partitioner.Partition<UniqueValueCountAppender<V>>> definePartitions(Collection<com.datatorrent.api.Partitioner.Partition<UniqueValueCountAppender<V>>> partitions, PartitioningContext context)
- {
- final int finalCapacity = DefaultPartition.getRequiredPartitionCount(context, this.partitionCount);
- UniqueValueCountAppender<V> anOldOperator = partitions.iterator().next().getPartitionedInstance();
- partitions.clear();
-
- Collection<Partition<UniqueValueCountAppender<V>>> newPartitions = Lists.newArrayListWithCapacity(finalCapacity);
-
- for (int i = 0; i < finalCapacity; i++) {
- try {
- @SuppressWarnings("unchecked")
- UniqueValueCountAppender<V> statefulUniqueCount = this.getClass().newInstance();
- DefaultPartition<UniqueValueCountAppender<V>> partition = new DefaultPartition<UniqueValueCountAppender<V>>(statefulUniqueCount);
- newPartitions.add(partition);
- } catch (Throwable cause) {
- DTThrowable.rethrow(cause);
- }
- }
-
- DefaultPartition.assignPartitionKeys(Collections.unmodifiableCollection(newPartitions), input);
- int lPartitionMask = newPartitions.iterator().next().getPartitionKeys().get(input).mask;
-
- for (Partition<UniqueValueCountAppender<V>> statefulUniqueCountPartition : newPartitions) {
- UniqueValueCountAppender<V> statefulUniqueCountInstance = statefulUniqueCountPartition.getPartitionedInstance();
-
- statefulUniqueCountInstance.partitionKeys = statefulUniqueCountPartition.getPartitionKeys().get(input).partitions;
- statefulUniqueCountInstance.partitionMask = lPartitionMask;
- statefulUniqueCountInstance.store = anOldOperator.store;
- statefulUniqueCountInstance.tableName = anOldOperator.tableName;
- statefulUniqueCountInstance.cacheManager = anOldOperator.cacheManager;
- }
- return newPartitions;
- }
-
- @Override
- public void partitioned(Map<Integer, com.datatorrent.api.Partitioner.Partition<UniqueValueCountAppender<V>>> partitions)
- {
- }
-
- private static final Logger LOGGER = LoggerFactory.getLogger(UniqueValueCountAppender.class);
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/distributedistinct/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/demos/distributedistinct/src/main/resources/META-INF/properties.xml b/demos/distributedistinct/src/main/resources/META-INF/properties.xml
deleted file mode 100644
index 8742328..0000000
--- a/demos/distributedistinct/src/main/resources/META-INF/properties.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-<?xml version="1.0"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<configuration>
- <!--
- <property>
- <name>dt.application.{appName}.operator.{opName}.prop.{propName}</name>
- <value>some-default-value (if value is not specified, it is required from the user or custom config when launching)</value>
- </property>
- -->
-</configuration>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/distributedistinct/src/site/conf/my-app-conf1.xml
----------------------------------------------------------------------
diff --git a/demos/distributedistinct/src/site/conf/my-app-conf1.xml b/demos/distributedistinct/src/site/conf/my-app-conf1.xml
deleted file mode 100644
index f35873b..0000000
--- a/demos/distributedistinct/src/site/conf/my-app-conf1.xml
+++ /dev/null
@@ -1,27 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<configuration>
- <property>
- <name>dt.attr.MASTER_MEMORY_MB</name>
- <value>1024</value>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/distributedistinct/src/test/java/com/datatorrent/demos/distributeddistinct/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/demos/distributedistinct/src/test/java/com/datatorrent/demos/distributeddistinct/ApplicationTest.java b/demos/distributedistinct/src/test/java/com/datatorrent/demos/distributeddistinct/ApplicationTest.java
deleted file mode 100644
index ef5473f..0000000
--- a/demos/distributedistinct/src/test/java/com/datatorrent/demos/distributeddistinct/ApplicationTest.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.distributeddistinct;
-
-import org.junit.Test;
-
-import com.datatorrent.api.LocalMode;
-
-public class ApplicationTest
-{
- @Test
- public void testSomeMethod() throws Exception
- {
- LocalMode.runApp(new Application(), 15000);
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/distributedistinct/src/test/java/com/datatorrent/demos/distributeddistinct/DistributedDistinctTest.java
----------------------------------------------------------------------
diff --git a/demos/distributedistinct/src/test/java/com/datatorrent/demos/distributeddistinct/DistributedDistinctTest.java b/demos/distributedistinct/src/test/java/com/datatorrent/demos/distributeddistinct/DistributedDistinctTest.java
deleted file mode 100644
index e013217..0000000
--- a/demos/distributedistinct/src/test/java/com/datatorrent/demos/distributeddistinct/DistributedDistinctTest.java
+++ /dev/null
@@ -1,200 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.distributeddistinct;
-
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Properties;
-import java.util.Set;
-
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.datatorrent.api.Context.OperatorContext;
-import com.datatorrent.api.DAG;
-import com.datatorrent.lib.algo.UniqueValueCount.InternalCountOutput;
-import com.datatorrent.lib.helper.OperatorContextTestHelper;
-import com.datatorrent.lib.helper.OperatorContextTestHelper.TestIdOperatorContext;
-
-/**
- * Test for {@link IntegerUniqueValueCountAppender} and {@link UniqueValueCountAppender}
- */
-public class DistributedDistinctTest
-{
- private static final Logger logger = LoggerFactory.getLogger(DistributedDistinctTest.class);
-
- private static final String APP_ID = "DistributedDistinctTest";
- private static final int OPERATOR_ID = 0;
-
- public static final String INMEM_DB_URL = "jdbc:hsqldb:mem:test;sql.syntax_mys=true";
- public static final String INMEM_DB_DRIVER = "org.hsqldb.jdbc.JDBCDriver";
- public static final String TABLE_NAME = "Test_Lookup_Cache";
-
- private static IntegerUniqueValueCountAppender valueCounter;
- private static String applicationPath;
-
- @Test
- public void testProcess() throws Exception
- {
- insertValues();
- Statement stmt = valueCounter.getStore().getConnection().createStatement();
-
- ResultSet resultSet = stmt.executeQuery("SELECT col2 FROM " + TABLE_NAME + " WHERE col1 = 1");
- ArrayList<Integer> answersOne = new ArrayList<Integer>();
- for (int i = 1; i < 16; i++) {
- answersOne.add(i);
- }
- Assert.assertEquals(answersOne, processResult(resultSet));
-
- resultSet = stmt.executeQuery("SELECT col2 FROM " + TABLE_NAME + " WHERE col1 = 2");
- ArrayList<Integer> answersTwo = new ArrayList<Integer>();
- answersTwo.add(3);
- answersTwo.add(6);
- answersTwo.add(9);
- for (int i = 11; i < 21; i++) {
- answersTwo.add(i);
- }
- Assert.assertEquals(answersTwo, processResult(resultSet));
-
- resultSet = stmt.executeQuery("SELECT col2 FROM " + TABLE_NAME + " WHERE col1 = 3");
- ArrayList<Integer> answersThree = new ArrayList<Integer>();
- answersThree.add(2);
- answersThree.add(4);
- answersThree.add(6);
- answersThree.add(8);
- answersThree.add(10);
- for (int i = 11; i < 21; i++) {
- answersThree.add(i);
- }
- Assert.assertEquals(answersThree, processResult(resultSet));
-
- valueCounter.teardown();
- }
-
- public static void insertValues()
- {
- logger.debug("start round 0");
- valueCounter.beginWindow(0);
- emitKeyVals(1, 1, 10, 1);
- emitKeyVals(1, 5, 15, 1);
- valueCounter.endWindow();
- logger.debug("end round 0");
-
- logger.debug("start round 1");
- valueCounter.beginWindow(1);
- emitKeyVals(2, 3, 15, 3);
- emitKeyVals(3, 2, 20, 2);
- emitKeyVals(3, 11, 20, 1);
- valueCounter.endWindow();
- logger.debug("end round 1");
-
- logger.debug("start round 2");
- valueCounter.beginWindow(2);
- emitKeyVals(3, 2, 20, 2);
- emitKeyVals(2, 11, 20, 1);
- valueCounter.endWindow();
- logger.debug("end round 2");
- }
-
- public static ArrayList<Integer> processResult(ResultSet resultSet)
- {
- ArrayList<Integer> tempList = new ArrayList<Integer>();
- try {
- while (resultSet.next()) {
- tempList.add(resultSet.getInt(1));
- }
- } catch (SQLException e) {
- throw new RuntimeException(e);
- }
- Collections.sort(tempList);
- return tempList;
- }
-
- public static void emitKeyVals(int key, int start, int end, int increment)
- {
- int count = 0;
- Set<Object> valSet = new HashSet<Object>();
- for (int i = start; i <= end; i += increment) {
- count++;
- valSet.add(i);
- }
- valueCounter.processTuple(new InternalCountOutput<Integer>(key, count, valSet));
- }
-
- @Test
- public void testSetup() throws Exception
- {
- insertValues();
- Statement stmt = valueCounter.getStore().getConnection().createStatement();
- com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap attributes = new com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap();
- attributes.put(DAG.APPLICATION_ID, APP_ID);
- attributes.put(DAG.APPLICATION_PATH, applicationPath);
- attributes.put(OperatorContext.ACTIVATION_WINDOW_ID, 2L);
-
- valueCounter.setup(new OperatorContextTestHelper.TestIdOperatorContext(0, attributes));
-
- ResultSet resultSet = stmt.executeQuery("SELECT col2 FROM " + TABLE_NAME + " WHERE col1 = 2");
- ArrayList<Integer> answersAfterClear = new ArrayList<Integer>();
- for (int i = 3; i < 16; i += 3) {
- answersAfterClear.add(i);
- }
- Assert.assertEquals(answersAfterClear, processResult(resultSet));
-
- resultSet = stmt.executeQuery("SELECT col2 FROM " + TABLE_NAME + " WHERE col1 = 3");
- ArrayList<Integer> answersThree = new ArrayList<Integer>();
- answersThree.add(2);
- answersThree.add(4);
- answersThree.add(6);
- answersThree.add(8);
- answersThree.add(10);
- for (int i = 11; i < 21; i++) {
- answersThree.add(i);
- }
- Assert.assertEquals(answersThree, processResult(resultSet));
- stmt.executeQuery("DELETE FROM " + TABLE_NAME);
- }
-
- @BeforeClass
- public static void setup() throws Exception
- {
- valueCounter = new IntegerUniqueValueCountAppender();
- Class.forName(INMEM_DB_DRIVER).newInstance();
- Connection con = DriverManager.getConnection(INMEM_DB_URL, new Properties());
- Statement stmt = con.createStatement();
- stmt.execute("CREATE TABLE IF NOT EXISTS " + TABLE_NAME + " (col1 INTEGER, col2 INTEGER, col3 BIGINT)");
- com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap attributes = new com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap();
- attributes.put(DAG.APPLICATION_ID, APP_ID);
- attributes.put(DAG.APPLICATION_PATH, applicationPath);
- attributes.put(OperatorContext.ACTIVATION_WINDOW_ID, 0L);
- valueCounter.setTableName(TABLE_NAME);
- valueCounter.getStore().setDatabaseDriver(INMEM_DB_DRIVER);
- valueCounter.getStore().setDatabaseUrl(INMEM_DB_URL);
- TestIdOperatorContext context = new OperatorContextTestHelper.TestIdOperatorContext(OPERATOR_ID, attributes);
- valueCounter.setup(context);
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/demos/distributedistinct/src/test/java/com/datatorrent/demos/distributeddistinct/StatefulApplicationTest.java
----------------------------------------------------------------------
diff --git a/demos/distributedistinct/src/test/java/com/datatorrent/demos/distributeddistinct/StatefulApplicationTest.java b/demos/distributedistinct/src/test/java/com/datatorrent/demos/distributeddistinct/StatefulApplicationTest.java
deleted file mode 100644
index 57ac964..0000000
--- a/demos/distributedistinct/src/test/java/com/datatorrent/demos/distributeddistinct/StatefulApplicationTest.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.datatorrent.demos.distributeddistinct;
-
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.Properties;
-
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.LocalMode;
-
-public class StatefulApplicationTest
-{
-
- @BeforeClass
- public static void setup()
- {
- try {
- Class.forName(StatefulUniqueCountTest.INMEM_DB_DRIVER).newInstance();
- Connection con = DriverManager.getConnection(StatefulUniqueCountTest.INMEM_DB_URL, new Properties());
- Statement stmt = con.createStatement();
- stmt.execute("CREATE TABLE IF NOT EXISTS " + StatefulUniqueCountTest.TABLE_NAME + " (col1 INTEGER, col2 INTEGER, col3 BIGINT)");
- } catch (InstantiationException e) {
- throw new RuntimeException(e);
- } catch (IllegalAccessException e) {
- throw new RuntimeException(e);
- } catch (ClassNotFoundException e) {
- throw new RuntimeException(e);
- } catch (SQLException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Test
- public void testApplication() throws Exception
- {
- LocalMode lma = LocalMode.newInstance();
- Configuration conf = new Configuration(false);
- conf.set("dt.operator.StatefulUniqueCounter.prop.tableName", "Test_Lookup_Cache");
- conf.set("dt.operator.StatefulUniqueCounter.prop.store.dbUrl", "jdbc:hsqldb:mem:test;sql.syntax_mys=true");
- conf.set("dt.operator.StatefulUniqueCounter.prop.store.dbDriver", "org.hsqldb.jdbcDriver");
-
- lma.prepareDAG(new StatefulApplication(), conf);
- lma.cloneDAG();
- LocalMode.Controller lc = lma.getController();
- lc.setHeartbeatMonitoringEnabled(false);
- lc.runAsync();
-
- long now = System.currentTimeMillis();
- while (System.currentTimeMillis() - now < 15000) {
- Thread.sleep(1000);
- }
-
- lc.shutdown();
- }
-}